use clap::Parser; use parse_wiki_text::{Node, Parameter}; use std::fs::File; use std::io::{stderr, stdin, stdout, BufWriter, Read, Write}; use std::sync::{Arc, Mutex}; use std::thread; use tar::Header; extern crate bzip2; extern crate parse_mediawiki_dump; #[derive(Parser)] struct Args { /// Stop after n articles (for debugging) #[arg(short, long)] limit: Option, /// Read bzip2 compressed input #[arg(short, long)] bzip2: bool, /// Stream tar to stdout #[arg(short, long)] tar: bool, /// Show non-fatal warnings #[arg(short, long)] verbose: bool, #[arg(short, long)] show_todo: bool, #[arg(short, long, default_value = "")] footer: String, } struct Config { show_todo: bool, } fn main() { let args = Args::parse(); let mut input: Box = Box::new(stdin()); if args.bzip2 { input = Box::new(bzip2::read::BzDecoder::new(input)) } let input = std::io::BufReader::new(input); let mut archive = tar::Builder::new(BufWriter::new(stdout())); let footer = format!( "staticwiki; {}", args.footer ); let progress = Arc::new(Mutex::new((0, String::new()))); { let progress = progress.clone(); thread::spawn(move || loop { let (i, name) = progress.lock().unwrap().clone(); eprint!("\r\x1b[2Karticles processed: {i}, current: {name}",); stderr().flush().unwrap(); std::thread::sleep(std::time::Duration::from_millis(1000 / 30)); }); } let config = Config { show_todo: args.show_todo, }; for (i, result) in parse_mediawiki_dump::parse(input).enumerate() { match result { Ok(page) => { *progress.lock().unwrap() = (i, page.title.clone()); if page.namespace == 0 && match &page.format { None => false, Some(format) => format == "text/x-wiki", } && match &page.model { None => false, Some(model) => model == "wikitext", } { let filename = urlencode(&page.title); let ast = parse_wiki_text::Configuration::default().parse(&page.text); if args.verbose { for w in ast.warnings { eprintln!("wikitext warning: {}", w.message.message()) } } use std::fmt::Write; let mut refs = vec![]; let mut html = String::from(""); write!(&mut html, "{}", escape(&page.title)).unwrap(); write!(&mut html, "

{}

", escape(&page.title)).unwrap(); render_toc(&mut html, &ast.nodes, &config); render_nodes(&mut html, &mut refs, &ast.nodes, &config); render_refs(&mut html, &refs); write!(&mut html, "").unwrap(); write!(&mut html, "").unwrap(); if args.tar { let mut header = Header::new_gnu(); header.set_size(html.as_bytes().len() as u64); header.set_cksum(); if let Err(e) = archive.append_data(&mut header, filename, html.as_bytes()) { eprintln!("could not append to archive: {e}") } } else { match File::create(format!("out/{}", filename)) { Ok(mut f) => f.write_all(html.as_bytes()).unwrap(), Err(e) => eprintln!("could not create file: {e}"), } } } else if args.verbose { eprintln!("page ignored: {:?}", page.title); } } Err(error) => { eprintln!("xml format error: {}", error); break; } } if Some(i) == args.limit { break; } } archive.finish().unwrap(); } fn urlencode(t: &str) -> String { t.replace("/", "_") } pub fn escape(text: &str) -> String { text.replace("&", "&") .replace("<", "<") .replace(">", ">") .replace("'", "’") .replace("\"", """) } fn render_nodes(html: &mut String, refs: &mut Vec, nodes: &Vec, config: &Config) { for n in nodes { render_node(html, refs, n, config) } } fn render_nodes_to_string(nodes: &Vec, refs: &mut Vec, config: &Config) -> String { let mut html = String::new(); render_nodes(&mut html, refs, nodes, config); return html; } fn render_node(html: &mut String, refs: &mut Vec, n: &Node, config: &Config) { use std::fmt::Write; match n { parse_wiki_text::Node::Bold { .. } => (), parse_wiki_text::Node::BoldItalic { .. } => (), parse_wiki_text::Node::Category { ordinal: _, target: _, .. } => { if config.show_todo { write!(html, "[todo: category]").unwrap() } } parse_wiki_text::Node::CharacterEntity { character: _, .. } => { if config.show_todo { write!(html, "[todo: character]").unwrap() } } parse_wiki_text::Node::Comment { .. } => (), parse_wiki_text::Node::DefinitionList { items: _, .. } => { if config.show_todo { write!(html, "[todo: def list]").unwrap() } } parse_wiki_text::Node::EndTag { name: _, .. } => (), //write!(html, "[todo: tag end]").unwrap(), parse_wiki_text::Node::StartTag { name: _, .. } => (), // write!(html, "[todo: start tag]").unwrap() parse_wiki_text::Node::ExternalLink { nodes: _, .. } => { if config.show_todo { write!(html, "[todo: external link]").unwrap() } } parse_wiki_text::Node::Heading { level, nodes, .. } => { let h = render_nodes_to_string(nodes, refs, config); write!(html, "{}", urlencode(&h), h).unwrap(); } parse_wiki_text::Node::HorizontalDivider { .. } => write!(html, "

").unwrap(), parse_wiki_text::Node::Image { target: _, text: _, .. } => write!(html, "[todo: image]").unwrap(), parse_wiki_text::Node::Italic { .. } => (), parse_wiki_text::Node::Link { target, text, .. } => write!( html, "{}", urlencode(target), // TODO does this always link to wikipedia? render_nodes_to_string(text, refs, config) ) .unwrap(), parse_wiki_text::Node::MagicWord { .. } => { if config.show_todo { write!(html, "[todo: magic]").unwrap() } } parse_wiki_text::Node::OrderedList { items, .. } => write!( html, "

{}", items .iter() .map(|e| format!( "

{}

", render_nodes_to_string(&e.nodes, refs, config) )) .collect::>() .join("") ) .unwrap(), parse_wiki_text::Node::UnorderedList { items, .. } => write!( html, "

{}", items .iter() .map(|e| format!( "

{}

", render_nodes_to_string(&e.nodes, refs, config) )) .collect::>() .join("") ) .unwrap(), parse_wiki_text::Node::ParagraphBreak { .. } => write!(html, "

").unwrap(), parse_wiki_text::Node::Parameter { default: _, name: _, .. } => { if config.show_todo { write!(html, "[todo: parameter]").unwrap() } } parse_wiki_text::Node::Preformatted { nodes, .. } => write!( html, "

{}

", render_nodes_to_string(nodes, refs, config) ) .unwrap(), parse_wiki_text::Node::Redirect { target, .. } => write!( html, "Redirect: {}", urlencode(target), urlencode(target) ) .unwrap(), parse_wiki_text::Node::Table { attributes: _, captions: _, rows: _, .. } => write!(html, "[todo: table]").unwrap(), parse_wiki_text::Node::Tag { name, nodes, .. } => match name.as_ref() { "ref" => { if !nodes.is_empty() { let r = render_nodes_to_string(nodes, refs, config); refs.push(r); let refid = refs.len(); write!(html, "^[{}]", refid, refid).unwrap(); } } "math" => { let r = render_nodes_to_string(nodes, refs, config); write!(html, "math: {}", r).unwrap(); } "syntaxhighlight" => { let r = render_nodes_to_string(nodes, refs, config); write!(html, "

{}

", r).unwrap(); } _ => { if config.show_todo { write!(html, "[todo: {name:?} tag]").unwrap() } } }, parse_wiki_text::Node::Template { name, parameters, .. } => { let name = match name.first() { Some(Node::Text { value, .. }) => value, _ => "", }; render_template(html, refs, name, parameters, config); } parse_wiki_text::Node::Text { value, .. } => write!(html, "{}", escape(value)).unwrap(), } } fn render_toc(html: &mut String, nodes: &Vec, config: &Config) { use std::fmt::Write; write!(html, "

").unwrap(); let mut k = 0; for n in nodes { match n { Node::Heading { level, nodes, .. } => { let level = *level as usize; while k < level { k += 1; write!(html, "

").unwrap(); } while k > level { k -= 1; write!(html, "").unwrap(); } let h = render_nodes_to_string(nodes, &mut vec![], config); write!(html, "

{}

", urlencode(&h), h).unwrap(); } _ => (), } } write!(html, "

").unwrap(); } fn render_template( html: &mut String, refs: &mut Vec, name: &str, params: &Vec, config: &Config, ) -> Option<()> { use std::fmt::Write; match name.to_lowercase().as_str() { "internetquelle" | "literatur" | "citation" | "man" | "external link" => { generic_template(html, name, params, config) } s if s.starts_with("cite ") => generic_template(html, name, params, config), // TODO this can panic "lang" | "transliteration" => write!( html, "{}", render_nodes_to_string(¶ms.get(1)?.value, refs, config) ) .unwrap(), s if s.starts_with("IPA") => write!( html, "{}", render_nodes_to_string(¶ms.get(0)?.value, refs, config) ) .unwrap(), "Siehe auch" | "Hauptartikel" | "main" => { let k = text_node(params.get(0)?.value.get(0)?); write!( html, "{}: {}", escape(name), urlencode(&k), escape(&k) ) .unwrap(); } "sfn" => {} _ => { if config.show_todo { write!(html, "[todo: {name:?} template]").unwrap(); } // write!(html, "[todo: {name:?} template

{params:#?}

]").unwrap(); // eprintln!("unsupported template {name:?}"); // eprintln!("{params:?}"); } } Some(()) } fn generic_template(html: &mut String, name: &str, params: &Vec, config: &Config) { use std::fmt::Write; write!(html, "{}:

{}: {}
{}: {}

").unwrap(); } pub fn text_node(n: &Node) -> String { match n { Node::Text { value, .. } => value.to_string(), _ => String::from("[todo: fix this bug :) ]"), } } fn render_refs(html: &mut String, refs: &Vec) { use std::fmt::Write; write!(html, "

").unwrap(); }

{}

Table of contents