//! Turning repository files into page content: markdown rendering with //! links resolved to the site's own file and raw views, and deciding how //! anything else should be shown. use std::borrow::Cow; use anyhow::Error; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; use pulldown_cmark::{Options, Parser, html}; use crate::repos::{File, Repo, RepoName, RepoPath}; // Files above this size are offered for download rather than shown. pub const MAX_SHOWN_BYTES: usize = 1024 * 1024; // Element ids from a document are prefixed so that they can never collide // with the page's own ids or clobber a DOM property; links to them are // prefixed the same way. const ID_PREFIX: &str = "user-content-"; // Everything in a path segment that is not safe in a URL path is encoded; // the separator is kept. const PATH_ENCODE: &AsciiSet = &NON_ALPHANUMERIC .remove(b'/') .remove(b'-') .remove(b'.') .remove(b'_') .remove(b'~'); /// The two views of a path in a repository: the page about it, and its /// bytes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum View { File, Raw, } impl View { fn segment(self) -> &'static str { match self { View::File => "file", View::Raw => "raw", } } } /// The URL of a repository's own page. pub fn repo_url(repo: &RepoName) -> String { format!("/{repo}") } /// Where links found in a document should point. Relative links are /// resolved against the document's own directory, the way they would be /// in a checkout, and land on the file view; images land on the raw view /// so that browsers receive the bytes. #[derive(Debug, Clone)] pub struct Links { repo: RepoName, dir: RepoPath, } impl Links { /// Links for a document at `path` in repository `repo`. pub fn for_document(repo: &RepoName, path: &RepoPath) -> Self { Links { repo: repo.clone(), dir: path.parent(), } } pub fn repo(&self) -> &RepoName { &self.repo } /// The URL of the repository's own page. pub fn repo_url(&self) -> String { repo_url(&self.repo) } /// The URL of a view of a path. pub fn url(&self, view: View, path: &RepoPath) -> String { let base = format!("/{}/{}", self.repo, view.segment()); if path.is_root() { base } else { format!("{base}/{}", encode_path(path)) } } /// Rewrites a link target from a document to the given view, or /// returns it unchanged when it is not a path into the repository. pub fn rewrite<'t>(&self, view: View, target: &'t str) -> Cow<'t, str> { match self.resolve(target) { Some((path, suffix)) => Cow::Owned(format!("{}{suffix}", self.url(view, &path))), None => Cow::Borrowed(target), } } // Rewrites one attribute of the sanitised HTML: link targets go to the // file view, image sources to the raw view, and in-page anchors follow // their prefixed ids. fn attribute<'v>(&self, element: &str, attribute: &str, value: &'v str) -> Cow<'v, str> { match (element, attribute) { ("a", "href") if value.starts_with('#') => { Cow::Owned(format!("#{ID_PREFIX}{}", &value[1..])) } ("a", "href") => self.rewrite(View::File, value), ("img", "src") => self.rewrite(View::Raw, value), _ => Cow::Borrowed(value), } } // Resolves a target to a repository path plus any `?query#fragment` // suffix, or None when the target is not a path into the repository. fn resolve(&self, target: &str) -> Option<(RepoPath, String)> { if target.is_empty() || !is_repository_relative(target) { return None; } let (path_part, suffix) = match target.find(['?', '#']) { Some(at) => (&target[..at], &target[at..]), None => (target, ""), }; let decoded = percent_decode_str(path_part).decode_utf8().ok()?; let mut resolved = if decoded.starts_with('/') { RepoPath::root() } else { self.dir.clone() }; for component in decoded.split('/') { match component { "" | "." => {} ".." => resolved = resolved.parent(), c => resolved = resolved.join(c), } } Some((resolved, suffix.to_owned())) } } // A target is a path into the repository unless it has a scheme, is // protocol-relative, or only addresses the current page. fn is_repository_relative(target: &str) -> bool { if target.starts_with("//") || target.starts_with('#') || target.starts_with('?') { return false; } let has_scheme = target.split_once(':').is_some_and(|(scheme, _)| { let mut chars = scheme.chars(); chars.next().is_some_and(|c| c.is_ascii_alphabetic()) && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) }); !has_scheme } fn encode_path(path: &RepoPath) -> String { utf8_percent_encode(&path.to_string(), PATH_ENCODE).to_string() } /// Renders CommonMark plus the GitHub-style extensions people actually use /// in READMEs, with links resolved and the result sanitised so a document /// cannot inject script. The sanitiser sees every URL, whether it came /// from markdown syntax or from raw HTML in the document, so links are /// resolved there and nowhere else. pub fn markdown(text: &str, links: &Links) -> String { let options = Options::ENABLE_TABLES | Options::ENABLE_FOOTNOTES | Options::ENABLE_STRIKETHROUGH | Options::ENABLE_TASKLISTS | Options::ENABLE_HEADING_ATTRIBUTES | Options::ENABLE_SMART_PUNCTUATION; let mut unsafe_html = String::new(); html::push_html(&mut unsafe_html, Parser::new_ext(text, options)); let links = links.clone(); ammonia::Builder::default() // Task list items are inert checkboxes. .add_tags(["input"]) .set_tag_attribute_value("input", "type", "checkbox") .set_tag_attribute_value("input", "disabled", "") .add_tag_attribute_values("input", "checked", [""]) // Heading and footnote ids, kept apart from the page's own. .add_generic_attributes(["id"]) .id_prefix(Some(ID_PREFIX)) .add_allowed_classes("sup", ["footnote-reference", "footnote-definition-label"]) .add_allowed_classes("div", ["footnote-definition"]) .attribute_filter(move |element, attribute, value| { Some(links.attribute(element, attribute, value)) }) .clean(&unsafe_html) .to_string() } /// Whether a file name denotes markdown. pub fn is_markdown(file_name: &str) -> bool { let lower = file_name.to_ascii_lowercase(); lower.ends_with(".md") || lower.ends_with(".markdown") } /// How a file should be presented on a page. #[derive(Debug, Clone)] pub enum Presentation { Markdown(String), Text(String), Image, Binary, TooLarge, } impl Presentation { /// Decides how to show a file, reading its bytes only when they are /// going to be shown. pub fn of(repo: &Repo, file: &File, links: &Links) -> Result { if file.size > MAX_SHOWN_BYTES { return Ok(Presentation::TooLarge); } let name = file.path.file_name().unwrap_or_default(); if named_type(name).is_some_and(|mime| mime.type_() == mime_guess::mime::IMAGE) { return Ok(Presentation::Image); } Ok(match String::from_utf8(repo.read(file)?) { Ok(text) if is_markdown(name) => Presentation::Markdown(markdown(&text, links)), Ok(text) => Presentation::Text(text), Err(_) => Presentation::Binary, }) } } /// The media type a file is served with from the raw view. Type is guessed /// from the name; a file with no telling name is plain text if `sample`, /// its first bytes (or all of them), is UTF-8, and application/octet-stream /// otherwise. pub fn mime_type(file_name: &str, sample: &[u8]) -> mime_guess::Mime { match named_type(file_name) { Some(mime) => mime, None if looks_like_text(sample) => mime_guess::mime::TEXT_PLAIN_UTF_8, None => mime_guess::mime::APPLICATION_OCTET_STREAM, } } // Whether a sample of a file is UTF-8, allowing for a character cut in // half at the end of it. fn looks_like_text(sample: &[u8]) -> bool { match std::str::from_utf8(sample) { Ok(_) => true, Err(err) => err.error_len().is_none(), } } // The media type a file name implies, or None when the name says nothing. // Text is always plain text, never HTML, and nothing is ever script or XML, // so that a hosted page cannot run on the site's origin; SVG is the one // XML type that is kept, because it is an image, and the raw view // sandboxes it. fn named_type(file_name: &str) -> Option { use mime_guess::mime; let guessed = mime_guess::from_path(file_name).first()?; if guessed == mime::IMAGE_SVG { return Some(guessed); } let active = guessed.type_() == mime::TEXT || guessed.subtype() == mime::JAVASCRIPT || guessed.subtype() == mime::XML || guessed.suffix() == Some(mime::XML); Some(if active { mime::TEXT_PLAIN_UTF_8 } else { guessed }) } #[cfg(test)] mod tests { use assert2::check; use super::*; fn links(doc: &str) -> Links { Links::for_document( &RepoName::parse("widget").unwrap(), &RepoPath::parse(doc).unwrap(), ) } #[test] fn relative_links_resolve_against_the_document_directory() { let l = links("docs/guide.md"); check!(l.rewrite(View::File, "intro.md") == "/widget/file/docs/intro.md"); check!(l.rewrite(View::File, "../LICENSE") == "/widget/file/LICENSE"); check!(l.rewrite(View::File, "./img/a.png") == "/widget/file/docs/img/a.png"); check!(l.rewrite(View::Raw, "img/a.png") == "/widget/raw/docs/img/a.png"); check!(l.rewrite(View::File, "../../../etc/passwd") == "/widget/file/etc/passwd"); } #[test] fn root_relative_links_start_at_the_repository_root() { let l = links("docs/guide.md"); check!(l.rewrite(View::File, "/LICENSE") == "/widget/file/LICENSE"); check!(l.url(View::File, &RepoPath::root()) == "/widget/file"); check!(l.repo_url() == "/widget"); } #[test] fn external_and_in_page_links_are_untouched() { let l = links("README.md"); for target in [ "https://example.com/x", "mailto:a@b.c", "//cdn.example.com/x.png", "#usage", "?page=2", ] { check!(l.rewrite(View::File, target) == target, "{target}"); } } #[test] fn queries_and_fragments_survive_and_names_are_encoded() { let l = links("README.md"); check!( l.rewrite(View::File, "docs/guide.md#install") == "/widget/file/docs/guide.md#install" ); check!(l.rewrite(View::File, "docs/my%20file.md") == "/widget/file/docs/my%20file.md"); check!(l.rewrite(View::File, "docs/my file.md") == "/widget/file/docs/my%20file.md"); } #[test] fn markdown_rewrites_links_and_images_and_strips_script() { let html = markdown( "[license](LICENSE) ![logo](docs/logo.png) raw \n\n x", &links("README.md"), ); check!(html.contains("href=\"/widget/file/LICENSE\"")); check!(html.contains("src=\"/widget/raw/docs/logo.png\"")); check!(html.contains("href=\"/widget/file/CHANGES\"")); check!(html.contains("src=\"/widget/raw/docs/big.png\" width=\"200\"")); check!(!html.contains("")); check!(html.contains("href=\"#user-content-usage\"")); let inputs: Vec<&str> = html .match_indices("').unwrap()]) .collect(); check!(inputs.len() == 2); check!( inputs .iter() .all(|tag| tag.contains("type=\"checkbox\"") && tag.contains("disabled=\"\"")) ); check!( inputs .iter() .filter(|tag| tag.contains("checked=\"\"")) .count() == 1 ); check!(html.contains("href=\"#user-content-1\"")); check!(html.contains("
")); } #[test] fn raw_media_types_never_execute() { check!(mime_type("index.html", b"

") == mime_guess::mime::TEXT_PLAIN_UTF_8); check!(mime_type("app.js", b"1") == mime_guess::mime::TEXT_PLAIN_UTF_8); check!(mime_type("app.mjs", b"1") == mime_guess::mime::TEXT_PLAIN_UTF_8); check!(mime_type("page.xhtml", b"1") == mime_guess::mime::TEXT_PLAIN_UTF_8); check!(mime_type("logo.png", b"\x89PNG") == mime_guess::mime::IMAGE_PNG); check!(mime_type("logo.svg", b"") == mime_guess::mime::IMAGE_SVG); check!(mime_type("LICENSE", b"MIT") == mime_guess::mime::TEXT_PLAIN_UTF_8); check!(mime_type("blob", b"\x00\xff") == mime_guess::mime::APPLICATION_OCTET_STREAM); let cut_short = &"caf\u{e9}".as_bytes()[..4]; check!(mime_type("notes", cut_short) == mime_guess::mime::TEXT_PLAIN_UTF_8); } }