Accept comma separated converter names, separate Atom from Html and Gemini converters.
Note: atom must be called separately now because of the separation. Example txt convert -t htm,atom xyz
This commit is contained in:
parent
4c32abf03b
commit
e4654aa652
38
cli/atom.ml
38
cli/atom.ml
|
@ -1,3 +1,5 @@
|
||||||
|
let ext = ".atom"
|
||||||
|
|
||||||
let esc = Converter.Html.esc
|
let esc = Converter.Html.esc
|
||||||
|
|
||||||
let element tag content = "<" ^ tag ^ ">" ^ content ^ "</" ^ tag ^ ">"
|
let element tag content = "<" ^ tag ^ ">" ^ content ^ "</" ^ tag ^ ">"
|
||||||
|
@ -44,14 +46,28 @@ let gmi_entry base_url text =
|
||||||
^ String_set.fold (fun elt a -> a ^ "<category term=\"" ^ elt ^ "\"/>") (Text.set "topics" text) ""
|
^ String_set.fold (fun elt a -> a ^ "<category term=\"" ^ elt ^ "\"/>") (Text.set "topics" text) ""
|
||||||
^ "</entry>\n"
|
^ "</entry>\n"
|
||||||
|
|
||||||
let feed title archive_id base_url alternate_type texts =
|
let base_url kv protocol = try
|
||||||
let entry, self = match alternate_type with
|
let locs = Logarion.Store.KV.find "Locations" kv in
|
||||||
| "text/gemini" -> gmi_entry, base_url^"/gmi.atom"
|
let _i = Str.(search_forward (regexp (protocol ^ "://[^;]*")) locs 0) in
|
||||||
| "text/html" | _ -> htm_entry, base_url^"/feed.atom" in
|
Str.(matched_string locs)
|
||||||
{|<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" xml:base="|} ^ base_url ^ {|"><title>|}
|
with Not_found -> Printf.eprintf "Missing location for %s" protocol; ""
|
||||||
^ title ^ {|</title><link rel="alternate" type="|} ^ alternate_type ^ {|" href="|}
|
|
||||||
^ base_url ^ {|/" /><link rel="self" type="application/atom+xml" href="|}
|
let indices alternate_type c =
|
||||||
^ self ^ {|" /><id>urn:uuid:|} ^ archive_id ^ "</id><updated>"
|
let file name = Logarion.File_store.file (Filename.concat c.Conversion.dir name) in
|
||||||
^ Logarion.Date.now () ^ "</updated>\n"
|
let title = try Logarion.Store.KV.find "Title" c.Conversion.kv with Not_found -> "" in
|
||||||
^ List.fold_left (fun acc t -> acc ^ entry base_url t) "" texts
|
let entry, fname, protocol_regexp = match alternate_type with
|
||||||
^ "</feed>"
|
| "text/gemini" -> gmi_entry, "gmi.atom", "gemini"
|
||||||
|
| "text/html" | _ -> htm_entry, "feed.atom", "https?"
|
||||||
|
in
|
||||||
|
let base_url = base_url c.kv protocol_regexp in
|
||||||
|
let self = Filename.concat base_url fname in
|
||||||
|
file fname @@
|
||||||
|
{|<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" xml:base="|} ^ base_url ^ {|"><title>|}
|
||||||
|
^ title ^ {|</title><link rel="alternate" type="|} ^ alternate_type ^ {|" href="|}
|
||||||
|
^ base_url ^ {|/" /><link rel="self" type="application/atom+xml" href="|}
|
||||||
|
^ self ^ {|" /><id>urn:uuid:|} ^ c.Conversion.id ^ "</id><updated>"
|
||||||
|
^ Logarion.Date.now () ^ "</updated>\n"
|
||||||
|
^ List.fold_left (fun acc t -> acc ^ entry base_url t) "" c.texts
|
||||||
|
^ "</feed>"
|
||||||
|
|
||||||
|
let converter format = Conversion.{ ext; page = None; indices = Some (indices format) }
|
||||||
|
|
|
@ -9,6 +9,6 @@ type t = {
|
||||||
|
|
||||||
type fn_t = {
|
type fn_t = {
|
||||||
ext: string;
|
ext: string;
|
||||||
page: t -> Logarion.Text.t -> string;
|
page: (t -> Logarion.Text.t -> string) option;
|
||||||
indices: t -> unit;
|
indices: (t -> unit) option;
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,20 +9,21 @@ let convert cs r (text, files) = match Text.str "Content-Type" text with
|
||||||
let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
|
let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
|
||||||
List.fold_left
|
List.fold_left
|
||||||
(fun a f ->
|
(fun a f ->
|
||||||
let dest = dest ^ f.Conversion.ext in
|
match f.Conversion.page with None -> false || a
|
||||||
if is_older source dest then (File_store.file dest (f.Conversion.page r text); true) else false
|
| Some page ->
|
||||||
|| a)
|
let dest = dest ^ f.Conversion.ext in
|
||||||
|
(if is_older source dest then (File_store.file dest (page r text); true) else false)
|
||||||
|
|| a)
|
||||||
false cs
|
false cs
|
||||||
| x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
|
| x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
|
||||||
|
|
||||||
let converters types kv =
|
let converters types kv =
|
||||||
|
let n = String.split_on_char ',' types in
|
||||||
let t = [] in
|
let t = [] in
|
||||||
let t = if ("htm" = types || "all" = types) then
|
let t = if List.(mem "all" n || mem "htm" n) then (Html.converter kv)::t else t in
|
||||||
(let htm = Html.init kv in
|
let t = if List.(mem "all" n || mem "atom" n) then (Atom.converter "text/html")::t else t in
|
||||||
Conversion.{ ext = Html.ext; page = Html.page htm; indices = Html.indices htm })::t
|
let t = if List.(mem "all" n || mem "gmi" n) then (Gemini.converter)::t else t in
|
||||||
else t in
|
let t = if List.(mem "all" n || mem "gmi-atom" n) then (Atom.converter "text/gemini")::t else t in
|
||||||
let t = if ("gmi" = types || "all" = types) then
|
|
||||||
Conversion.{ ext = Gemini.ext; page = Gemini.page; indices = Gemini.indices}::t else t in
|
|
||||||
t
|
t
|
||||||
|
|
||||||
let directory converters noindex dir id kv =
|
let directory converters noindex dir id kv =
|
||||||
|
@ -35,7 +36,7 @@ let directory converters noindex dir id kv =
|
||||||
let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" kv)
|
let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" kv)
|
||||||
with Not_found -> Topic_set.roots topics in
|
with Not_found -> Topic_set.roots topics in
|
||||||
let repo = Conversion.{ repo with topic_roots; topics; texts } in
|
let repo = Conversion.{ repo with topic_roots; topics; texts } in
|
||||||
if not noindex then List.iter (fun c -> c.Conversion.indices repo) converters;
|
if not noindex then List.iter (fun c -> match c.Conversion.indices with None -> () | Some f -> f repo) converters;
|
||||||
Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
|
Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
|
||||||
|
|
||||||
let at_path types noindex path =
|
let at_path types noindex path =
|
||||||
|
@ -47,8 +48,7 @@ let at_path types noindex path =
|
||||||
| Ok { info; peers; _ } ->
|
| Ok { info; peers; _ } ->
|
||||||
let kv = let f = Filename.concat dir ".convert.conf" in (* TODO: better place to store convert conf? *)
|
let kv = let f = Filename.concat dir ".convert.conf" in (* TODO: better place to store convert conf? *)
|
||||||
if Sys.file_exists f then File_store.of_kv_file f else Store.KV.empty in
|
if Sys.file_exists f then File_store.of_kv_file f else Store.KV.empty in
|
||||||
let kv = if Store.KV.mem "Title" kv then kv
|
let kv = if Store.KV.mem "Title" kv then kv else Store.KV.add "Title" info.Header_pack.title kv in
|
||||||
else Store.KV.add "Title" info.Header_pack.title kv in
|
|
||||||
let kv = Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
|
let kv = Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
|
||||||
let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
|
let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
|
||||||
let cs = converters types kv in
|
let cs = converters types kv in
|
||||||
|
|
|
@ -90,19 +90,11 @@ let indices r =
|
||||||
let file name = File_store.file (Filename.concat r.Conversion.dir name) in
|
let file name = File_store.file (Filename.concat r.Conversion.dir name) in
|
||||||
let index_name = try Store.KV.find "Gemini-index" r.kv with Not_found -> "index.gmi" in
|
let index_name = try Store.KV.find "Gemini-index" r.kv with Not_found -> "index.gmi" in
|
||||||
let title = try Store.KV.find "Title" r.Conversion.kv with Not_found -> "" in
|
let title = try Store.KV.find "Title" r.Conversion.kv with Not_found -> "" in
|
||||||
|
if index_name <> "" then file index_name (topic_main_index r title r.topic_roots r.texts);
|
||||||
if index_name <> "" then
|
|
||||||
file index_name (topic_main_index r title r.topic_roots r.texts);
|
|
||||||
|
|
||||||
file "index.date.gmi" (date_index title r.texts);
|
file "index.date.gmi" (date_index title r.texts);
|
||||||
|
|
||||||
List.iter
|
List.iter
|
||||||
(fun topic -> file ("index." ^ topic ^ ".gmi")
|
(fun topic -> file ("index." ^ topic ^ ".gmi")
|
||||||
(topic_sub_index title r.topics topic r.texts))
|
(topic_sub_index title r.topics topic r.texts))
|
||||||
r.topic_roots;
|
r.topic_roots
|
||||||
|
|
||||||
let base_url = try
|
let converter = Conversion.{ ext; page = Some page; indices = Some indices}
|
||||||
let _i = Str.(search_forward (regexp "gemini?://[^;]*") (Store.KV.find "Locations" r.kv) 0) in
|
|
||||||
Str.(matched_string (Store.KV.find "Locations" r.kv))
|
|
||||||
with Not_found -> prerr_endline "Missing location for Gemini"; "" in
|
|
||||||
file "gmi.atom" (Atom.feed title r.id base_url "text/gemini" r.texts)
|
|
||||||
|
|
30
cli/html.ml
30
cli/html.ml
|
@ -14,8 +14,8 @@ let init kv =
|
||||||
let footer = to_string "HTM-footer" kv in
|
let footer = to_string "HTM-footer" kv in
|
||||||
{ templates = { header; footer} }
|
{ templates = { header; footer} }
|
||||||
|
|
||||||
let wrap c htm text_title body =
|
let wrap conv htm text_title body =
|
||||||
let site_title = try Logarion.Store.KV.find "Title" c.Conversion.kv
|
let site_title = try Logarion.Store.KV.find "Title" conv.Conversion.kv
|
||||||
with Not_found -> "" in
|
with Not_found -> "" in
|
||||||
let replace x = let open Str in
|
let replace x = let open Str in
|
||||||
global_replace (regexp "{{archive-title}}") site_title x
|
global_replace (regexp "{{archive-title}}") site_title x
|
||||||
|
@ -57,7 +57,8 @@ let page htm conversion text =
|
||||||
let authors = (Person.Set.to_string text.authors ^ " ") in
|
let authors = (Person.Set.to_string text.authors ^ " ") in
|
||||||
let keywords = str_set "keywords" text in
|
let keywords = str_set "keywords" text in
|
||||||
let header =
|
let header =
|
||||||
let time x = {|<time datetime="|} ^ x ^ {|">|} ^ x ^ "</time>" in
|
let time x = Printf.sprintf {|<time datetime="%s">%s</time>|}
|
||||||
|
(Date.rfc_string x) (Date.pretty_date x) in
|
||||||
let topic_links x =
|
let topic_links x =
|
||||||
let to_linked t a =
|
let to_linked t a =
|
||||||
let ts = Topic_set.of_string t in
|
let ts = Topic_set.of_string t in
|
||||||
|
@ -66,7 +67,7 @@ let page htm conversion text =
|
||||||
"<article><header><dl>"
|
"<article><header><dl>"
|
||||||
^ opt_kv "Title:" text.title
|
^ opt_kv "Title:" text.title
|
||||||
^ opt_kv "Authors:" authors
|
^ opt_kv "Authors:" authors
|
||||||
^ opt_kv "Date: " (time (Date.(pretty_date @@ listing text.date)))
|
^ opt_kv "Date: " (time (Date.listing text.date))
|
||||||
^ opt_kv "Series: " (str_set "series" text)
|
^ opt_kv "Series: " (str_set "series" text)
|
||||||
^ opt_kv "Topics: " (topic_links (set "topics" text))
|
^ opt_kv "Topics: " (topic_links (set "topics" text))
|
||||||
^ opt_kv "Keywords: " keywords
|
^ opt_kv "Keywords: " keywords
|
||||||
|
@ -161,24 +162,15 @@ let topic_sub_index conv htm topic_map topic_root metas =
|
||||||
(* ^ {|<a href=".atom" id="feed">|}^ String.capitalize_ascii topic_root ^{| feed </a>|}*)
|
(* ^ {|<a href=".atom" id="feed">|}^ String.capitalize_ascii topic_root ^{| feed </a>|}*)
|
||||||
^ listing_index topic_map [topic_root] "" metas)
|
^ listing_index topic_map [topic_root] "" metas)
|
||||||
|
|
||||||
open Logarion
|
|
||||||
let indices htm c =
|
let indices htm c =
|
||||||
let file name = Logarion.File_store.file (Filename.concat c.Conversion.dir name) in
|
let file name = Logarion.File_store.file (Filename.concat c.Conversion.dir name) in
|
||||||
let index_name = try Store.KV.find "HTM-index" c.Conversion.kv with Not_found -> "index.html" in
|
let index_name = try Logarion.Store.KV.find "HTM-index" c.Conversion.kv with Not_found -> "index.html" in
|
||||||
let title = try Store.KV.find "Title" c.Conversion.kv with Not_found -> "" in
|
if index_name <> "" then file index_name (topic_main_index c htm c.topic_roots c.texts);
|
||||||
|
|
||||||
if index_name <> "" then
|
|
||||||
file index_name (topic_main_index c htm c.topic_roots c.texts);
|
|
||||||
|
|
||||||
file "index.date.htm" (date_index c htm c.texts);
|
file "index.date.htm" (date_index c htm c.texts);
|
||||||
|
|
||||||
List.iter
|
List.iter
|
||||||
(fun root -> file ("index." ^ root ^ ".htm") (topic_sub_index c htm c.topics root c.texts))
|
(fun root -> file ("index." ^ root ^ ".htm") (topic_sub_index c htm c.topics root c.texts))
|
||||||
c.topic_roots;
|
c.topic_roots
|
||||||
|
|
||||||
let base_url = try
|
let converter kv =
|
||||||
let locs = Store.KV.find "Locations" c.kv in
|
let htm = init kv in
|
||||||
let _i = Str.(search_forward (regexp "https?://[^;]*") locs 0) in
|
Conversion.{ ext; page = Some (page htm); indices = Some (indices htm) }
|
||||||
Str.(matched_string locs)
|
|
||||||
with Not_found -> prerr_endline "Missing location for HTTP(S)"; "" in
|
|
||||||
file "feed.atom" (Atom.feed title c.id base_url "text/html" c.texts)
|
|
||||||
|
|
Loading…
Reference in New Issue