render.rs
raw
//! Turning repository files into page content: markdown rendering with
//! links resolved to the site's own file and raw views, and deciding how
//! anything else should be shown.
use std::borrow::Cow;
use anyhow::Error;
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
use pulldown_cmark::{Options, Parser, html};
use crate::repos::{File, Repo, RepoName, RepoPath};
// Files above this size are offered for download rather than shown.
pub const MAX_SHOWN_BYTES: usize = 1024 * 1024;
// Element ids from a document are prefixed so that they can never collide
// with the page's own ids or clobber a DOM property; links to them are
// prefixed the same way.
const ID_PREFIX: &str = "user-content-";
// Everything in a path segment that is not safe in a URL path is encoded;
// the separator is kept.
const PATH_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'/')
.remove(b'-')
.remove(b'.')
.remove(b'_')
.remove(b'~');
/// The two views of a path in a repository: the page about it, and its
/// bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum View {
File,
Raw,
}
impl View {
fn segment(self) -> &'static str {
match self {
View::File => "file",
View::Raw => "raw",
}
}
}
/// The URL of a repository's own page.
pub fn repo_url(repo: &RepoName) -> String {
format!("/{repo}")
}
/// Where links found in a document should point. Relative links are
/// resolved against the document's own directory, the way they would be
/// in a checkout, and land on the file view; images land on the raw view
/// so that browsers receive the bytes.
#[derive(Debug, Clone)]
pub struct Links {
repo: RepoName,
dir: RepoPath,
}
impl Links {
/// Links for a document at `path` in repository `repo`.
pub fn for_document(repo: &RepoName, path: &RepoPath) -> Self {
Links {
repo: repo.clone(),
dir: path.parent(),
}
}
pub fn repo(&self) -> &RepoName {
&self.repo
}
/// The URL of the repository's own page.
pub fn repo_url(&self) -> String {
repo_url(&self.repo)
}
/// The URL of a view of a path.
pub fn url(&self, view: View, path: &RepoPath) -> String {
let base = format!("/{}/{}", self.repo, view.segment());
if path.is_root() {
base
} else {
format!("{base}/{}", encode_path(path))
}
}
/// Rewrites a link target from a document to the given view, or
/// returns it unchanged when it is not a path into the repository.
pub fn rewrite<'t>(&self, view: View, target: &'t str) -> Cow<'t, str> {
match self.resolve(target) {
Some((path, suffix)) => Cow::Owned(format!("{}{suffix}", self.url(view, &path))),
None => Cow::Borrowed(target),
}
}
// Rewrites one attribute of the sanitised HTML: link targets go to the
// file view, image sources to the raw view, and in-page anchors follow
// their prefixed ids.
fn attribute<'v>(&self, element: &str, attribute: &str, value: &'v str) -> Cow<'v, str> {
match (element, attribute) {
("a", "href") if value.starts_with('#') => {
Cow::Owned(format!("#{ID_PREFIX}{}", &value[1..]))
}
("a", "href") => self.rewrite(View::File, value),
("img", "src") => self.rewrite(View::Raw, value),
_ => Cow::Borrowed(value),
}
}
// Resolves a target to a repository path plus any `?query#fragment`
// suffix, or None when the target is not a path into the repository.
fn resolve(&self, target: &str) -> Option<(RepoPath, String)> {
if target.is_empty() || !is_repository_relative(target) {
return None;
}
let (path_part, suffix) = match target.find(['?', '#']) {
Some(at) => (&target[..at], &target[at..]),
None => (target, ""),
};
let decoded = percent_decode_str(path_part).decode_utf8().ok()?;
let mut resolved = if decoded.starts_with('/') {
RepoPath::root()
} else {
self.dir.clone()
};
for component in decoded.split('/') {
match component {
"" | "." => {}
".." => resolved = resolved.parent(),
c => resolved = resolved.join(c),
}
}
Some((resolved, suffix.to_owned()))
}
}
// A target is a path into the repository unless it has a scheme, is
// protocol-relative, or only addresses the current page.
fn is_repository_relative(target: &str) -> bool {
if target.starts_with("//") || target.starts_with('#') || target.starts_with('?') {
return false;
}
let has_scheme = target.split_once(':').is_some_and(|(scheme, _)| {
let mut chars = scheme.chars();
chars.next().is_some_and(|c| c.is_ascii_alphabetic())
&& chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
});
!has_scheme
}
fn encode_path(path: &RepoPath) -> String {
utf8_percent_encode(&path.to_string(), PATH_ENCODE).to_string()
}
/// Renders CommonMark plus the GitHub-style extensions people actually use
/// in READMEs, with links resolved and the result sanitised so a document
/// cannot inject script. The sanitiser sees every URL, whether it came
/// from markdown syntax or from raw HTML in the document, so links are
/// resolved there and nowhere else.
pub fn markdown(text: &str, links: &Links) -> String {
let options = Options::ENABLE_TABLES
| Options::ENABLE_FOOTNOTES
| Options::ENABLE_STRIKETHROUGH
| Options::ENABLE_TASKLISTS
| Options::ENABLE_HEADING_ATTRIBUTES
| Options::ENABLE_SMART_PUNCTUATION;
let mut unsafe_html = String::new();
html::push_html(&mut unsafe_html, Parser::new_ext(text, options));
let links = links.clone();
ammonia::Builder::default()
// Task list items are inert checkboxes.
.add_tags(["input"])
.set_tag_attribute_value("input", "type", "checkbox")
.set_tag_attribute_value("input", "disabled", "")
.add_tag_attribute_values("input", "checked", [""])
// Heading and footnote ids, kept apart from the page's own.
.add_generic_attributes(["id"])
.id_prefix(Some(ID_PREFIX))
.add_allowed_classes("sup", ["footnote-reference", "footnote-definition-label"])
.add_allowed_classes("div", ["footnote-definition"])
.attribute_filter(move |element, attribute, value| {
Some(links.attribute(element, attribute, value))
})
.clean(&unsafe_html)
.to_string()
}
/// Whether a file name denotes markdown.
pub fn is_markdown(file_name: &str) -> bool {
let lower = file_name.to_ascii_lowercase();
lower.ends_with(".md") || lower.ends_with(".markdown")
}
/// How a file should be presented on a page.
#[derive(Debug, Clone)]
pub enum Presentation {
Markdown(String),
Text(String),
Image,
Binary,
TooLarge,
}
impl Presentation {
/// Decides how to show a file, reading its bytes only when they are
/// going to be shown.
pub fn of(repo: &Repo, file: &File, links: &Links) -> Result<Self, Error> {
if file.size > MAX_SHOWN_BYTES {
return Ok(Presentation::TooLarge);
}
let name = file.path.file_name().unwrap_or_default();
if named_type(name).is_some_and(|mime| mime.type_() == mime_guess::mime::IMAGE) {
return Ok(Presentation::Image);
}
Ok(match String::from_utf8(repo.read(file)?) {
Ok(text) if is_markdown(name) => Presentation::Markdown(markdown(&text, links)),
Ok(text) => Presentation::Text(text),
Err(_) => Presentation::Binary,
})
}
}
/// The media type a file is served with from the raw view. Type is guessed
/// from the name; a file with no telling name is plain text if `sample`,
/// its first bytes (or all of them), is UTF-8, and application/octet-stream
/// otherwise.
pub fn mime_type(file_name: &str, sample: &[u8]) -> mime_guess::Mime {
match named_type(file_name) {
Some(mime) => mime,
None if looks_like_text(sample) => mime_guess::mime::TEXT_PLAIN_UTF_8,
None => mime_guess::mime::APPLICATION_OCTET_STREAM,
}
}
// Whether a sample of a file is UTF-8, allowing for a character cut in
// half at the end of it.
fn looks_like_text(sample: &[u8]) -> bool {
match std::str::from_utf8(sample) {
Ok(_) => true,
Err(err) => err.error_len().is_none(),
}
}
// The media type a file name implies, or None when the name says nothing.
// Text is always plain text, never HTML, and nothing is ever script or XML,
// so that a hosted page cannot run on the site's origin; SVG is the one
// XML type that is kept, because it is an image, and the raw view
// sandboxes it.
fn named_type(file_name: &str) -> Option<mime_guess::Mime> {
use mime_guess::mime;
let guessed = mime_guess::from_path(file_name).first()?;
if guessed == mime::IMAGE_SVG {
return Some(guessed);
}
let active = guessed.type_() == mime::TEXT
|| guessed.subtype() == mime::JAVASCRIPT
|| guessed.subtype() == mime::XML
|| guessed.suffix() == Some(mime::XML);
Some(if active {
mime::TEXT_PLAIN_UTF_8
} else {
guessed
})
}
#[cfg(test)]
mod tests {
use assert2::check;
use super::*;
fn links(doc: &str) -> Links {
Links::for_document(
&RepoName::parse("widget").unwrap(),
&RepoPath::parse(doc).unwrap(),
)
}
#[test]
fn relative_links_resolve_against_the_document_directory() {
let l = links("docs/guide.md");
check!(l.rewrite(View::File, "intro.md") == "/widget/file/docs/intro.md");
check!(l.rewrite(View::File, "../LICENSE") == "/widget/file/LICENSE");
check!(l.rewrite(View::File, "./img/a.png") == "/widget/file/docs/img/a.png");
check!(l.rewrite(View::Raw, "img/a.png") == "/widget/raw/docs/img/a.png");
check!(l.rewrite(View::File, "../../../etc/passwd") == "/widget/file/etc/passwd");
}
#[test]
fn root_relative_links_start_at_the_repository_root() {
let l = links("docs/guide.md");
check!(l.rewrite(View::File, "/LICENSE") == "/widget/file/LICENSE");
check!(l.url(View::File, &RepoPath::root()) == "/widget/file");
check!(l.repo_url() == "/widget");
}
#[test]
fn external_and_in_page_links_are_untouched() {
let l = links("README.md");
for target in [
"https://example.com/x",
"mailto:a@b.c",
"//cdn.example.com/x.png",
"#usage",
"?page=2",
] {
check!(l.rewrite(View::File, target) == target, "{target}");
}
}
#[test]
fn queries_and_fragments_survive_and_names_are_encoded() {
let l = links("README.md");
check!(
l.rewrite(View::File, "docs/guide.md#install") == "/widget/file/docs/guide.md#install"
);
check!(l.rewrite(View::File, "docs/my%20file.md") == "/widget/file/docs/my%20file.md");
check!(l.rewrite(View::File, "docs/my file.md") == "/widget/file/docs/my%20file.md");
}
#[test]
fn markdown_rewrites_links_and_images_and_strips_script() {
let html = markdown(
"[license](LICENSE)  <a href=\"CHANGES\">raw</a> <img src=\"docs/big.png\" width=\"200\">\n\n<script>alert(1)</script> <a href=\"javascript:alert(1)\">x</a>",
&links("README.md"),
);
check!(html.contains("href=\"/widget/file/LICENSE\""));
check!(html.contains("src=\"/widget/raw/docs/logo.png\""));
check!(html.contains("href=\"/widget/file/CHANGES\""));
check!(html.contains("src=\"/widget/raw/docs/big.png\" width=\"200\""));
check!(!html.contains("<script"));
check!(!html.contains("javascript:"));
}
#[test]
fn markdown_keeps_anchors_footnotes_and_task_lists_working() {
let html = markdown(
"# Usage {#usage}\n\n[u](#usage)\n\n- [x] done\n- [ ] todo\n\nfoot[^1]\n\n[^1]: note\n",
&links("README.md"),
);
check!(html.contains("<h1 id=\"user-content-usage\">"));
check!(html.contains("href=\"#user-content-usage\""));
let inputs: Vec<&str> = html
.match_indices("<input")
.map(|(at, _)| &html[at..at + html[at..].find('>').unwrap()])
.collect();
check!(inputs.len() == 2);
check!(
inputs
.iter()
.all(|tag| tag.contains("type=\"checkbox\"") && tag.contains("disabled=\"\""))
);
check!(
inputs
.iter()
.filter(|tag| tag.contains("checked=\"\""))
.count()
== 1
);
check!(html.contains("href=\"#user-content-1\""));
check!(html.contains("<div class=\"footnote-definition\" id=\"user-content-1\">"));
}
#[test]
fn raw_media_types_never_execute() {
check!(mime_type("index.html", b"<p>") == mime_guess::mime::TEXT_PLAIN_UTF_8);
check!(mime_type("app.js", b"1") == mime_guess::mime::TEXT_PLAIN_UTF_8);
check!(mime_type("app.mjs", b"1") == mime_guess::mime::TEXT_PLAIN_UTF_8);
check!(mime_type("page.xhtml", b"1") == mime_guess::mime::TEXT_PLAIN_UTF_8);
check!(mime_type("logo.png", b"\x89PNG") == mime_guess::mime::IMAGE_PNG);
check!(mime_type("logo.svg", b"<svg/>") == mime_guess::mime::IMAGE_SVG);
check!(mime_type("LICENSE", b"MIT") == mime_guess::mime::TEXT_PLAIN_UTF_8);
check!(mime_type("blob", b"\x00\xff") == mime_guess::mime::APPLICATION_OCTET_STREAM);
let cut_short = &"caf\u{e9}".as_bytes()[..4];
check!(mime_type("notes", cut_short) == mime_guess::mime::TEXT_PLAIN_UTF_8);
}
}