orbifx
/
logarion
1
1
Fork 0
Code Pull Requests Releases Activity

Accept comma separated converter names, separate Atom from Html and Gemini converters. 

Note: atom must be called separately now because of the separation. Example

txt convert -t htm,atom xyz
This commit is contained in:
orbifx 2022-10-26 20:36:02 +01:00
parent 4c32abf03b
commit e4654aa652
5 changed files with 55 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
cli/atom.ml
View File

@ -1,3 +1,5 @@
let ext = ".atom"
let esc = Converter.Html.esc
let element tag content = "<" ^ tag ^ ">" ^ content ^ "</" ^ tag ^ ">"
@ -44,14 +46,28 @@ let gmi_entry base_url text =
^ String_set.fold (fun elt a -> a ^ "<category term=\"" ^ elt ^ "\"/>") (Text.set "topics" text) ""
^ "</entry>\n"
let feed title archive_id base_url alternate_type texts =
let entry, self = match alternate_type with
| "text/gemini" -> gmi_entry, base_url^"/gmi.atom"
| "text/html" | _ -> htm_entry, base_url^"/feed.atom" in
{|<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" xml:base="|} ^ base_url ^ {|"><title>|}
^ title ^ {|</title><link rel="alternate" type="|} ^ alternate_type ^ {|" href="|}
^ base_url ^ {|/" /><link rel="self" type="application/atom+xml" href="|}
^ self ^ {|" /><id>urn:uuid:|} ^ archive_id ^ "</id><updated>"
^ Logarion.Date.now () ^ "</updated>\n"
^ List.fold_left (fun acc t -> acc ^ entry base_url t) "" texts
^ "</feed>"
let base_url kv protocol = try
let locs = Logarion.Store.KV.find "Locations" kv in
let _i = Str.(search_forward (regexp (protocol ^ "://[^;]*")) locs 0) in
Str.(matched_string locs)
with Not_found -> Printf.eprintf "Missing location for %s" protocol; ""
let indices alternate_type c =
let file name = Logarion.File_store.file (Filename.concat c.Conversion.dir name) in
let title = try Logarion.Store.KV.find "Title" c.Conversion.kv with Not_found -> "" in
let entry, fname, protocol_regexp = match alternate_type with
| "text/gemini" -> gmi_entry, "gmi.atom", "gemini"
| "text/html" | _ -> htm_entry, "feed.atom", "https?"
in
let base_url = base_url c.kv protocol_regexp in
let self = Filename.concat base_url fname in
file fname @@
{|<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" xml:base="|} ^ base_url ^ {|"><title>|}
^ title ^ {|</title><link rel="alternate" type="|} ^ alternate_type ^ {|" href="|}
^ base_url ^ {|/" /><link rel="self" type="application/atom+xml" href="|}
^ self ^ {|" /><id>urn:uuid:|} ^ c.Conversion.id ^ "</id><updated>"
^ Logarion.Date.now () ^ "</updated>\n"
^ List.fold_left (fun acc t -> acc ^ entry base_url t) "" c.texts
^ "</feed>"
let converter format = Conversion.{ ext; page = None; indices = Some (indices format) }

4
cli/conversion.ml
View File

@ -9,6 +9,6 @@ type t = {
type fn_t = {
ext: string;
page: t -> Logarion.Text.t -> string;
indices: t -> unit;
page: (t -> Logarion.Text.t -> string) option;
indices: (t -> unit) option;
}

24
cli/convert.ml
View File

@ -9,20 +9,21 @@ let convert cs r (text, files) = match Text.str "Content-Type" text with
let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
List.fold_left
(fun a f ->
let dest = dest ^ f.Conversion.ext in
if is_older source dest then (File_store.file dest (f.Conversion.page r text); true) else false
|| a)
match f.Conversion.page with None -> false || a
| Some page ->
let dest = dest ^ f.Conversion.ext in
(if is_older source dest then (File_store.file dest (page r text); true) else false)
|| a)
false cs
| x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
let converters types kv =
let n = String.split_on_char ',' types in
let t = [] in
let t = if ("htm" = types || "all" = types) then
(let htm = Html.init kv in
Conversion.{ ext = Html.ext; page = Html.page htm; indices = Html.indices htm })::t
else t in
let t = if ("gmi" = types || "all" = types) then
Conversion.{ ext = Gemini.ext; page = Gemini.page; indices = Gemini.indices}::t else t in
let t = if List.(mem "all" n || mem "htm" n) then (Html.converter kv)::t else t in
let t = if List.(mem "all" n || mem "atom" n) then (Atom.converter "text/html")::t else t in
let t = if List.(mem "all" n || mem "gmi" n) then (Gemini.converter)::t else t in
let t = if List.(mem "all" n || mem "gmi-atom" n) then (Atom.converter "text/gemini")::t else t in
t
let directory converters noindex dir id kv =
@ -35,7 +36,7 @@ let directory converters noindex dir id kv =
let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" kv)
with Not_found -> Topic_set.roots topics in
let repo = Conversion.{ repo with topic_roots; topics; texts } in
if not noindex then List.iter (fun c -> c.Conversion.indices repo) converters;
if not noindex then List.iter (fun c -> match c.Conversion.indices with None -> () | Some f -> f repo) converters;
Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
let at_path types noindex path =
@ -47,8 +48,7 @@ let at_path types noindex path =
| Ok { info; peers; _ } ->
let kv = let f = Filename.concat dir ".convert.conf" in (* TODO: better place to store convert conf? *)
if Sys.file_exists f then File_store.of_kv_file f else Store.KV.empty in
let kv = if Store.KV.mem "Title" kv then kv
else Store.KV.add "Title" info.Header_pack.title kv in
let kv = if Store.KV.mem "Title" kv then kv else Store.KV.add "Title" info.Header_pack.title kv in
let kv = Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
let cs = converters types kv in

14
cli/gemini.ml
View File

@ -90,19 +90,11 @@ let indices r =
let file name = File_store.file (Filename.concat r.Conversion.dir name) in
let index_name = try Store.KV.find "Gemini-index" r.kv with Not_found -> "index.gmi" in
let title = try Store.KV.find "Title" r.Conversion.kv with Not_found -> "" in
if index_name <> "" then
file index_name (topic_main_index r title r.topic_roots r.texts);
if index_name <> "" then file index_name (topic_main_index r title r.topic_roots r.texts);
file "index.date.gmi" (date_index title r.texts);
List.iter
(fun topic -> file ("index." ^ topic ^ ".gmi")
(topic_sub_index title r.topics topic r.texts))
r.topic_roots;
r.topic_roots
let base_url = try
let _i = Str.(search_forward (regexp "gemini?://[^;]*") (Store.KV.find "Locations" r.kv) 0) in
Str.(matched_string (Store.KV.find "Locations" r.kv))
with Not_found -> prerr_endline "Missing location for Gemini"; "" in
file "gmi.atom" (Atom.feed title r.id base_url "text/gemini" r.texts)
let converter = Conversion.{ ext; page = Some page; indices = Some indices}

30
cli/html.ml
View File

@ -14,8 +14,8 @@ let init kv =
let footer = to_string "HTM-footer" kv in
{ templates = { header; footer} }
let wrap c htm text_title body =
let site_title = try Logarion.Store.KV.find "Title" c.Conversion.kv
let wrap conv htm text_title body =
let site_title = try Logarion.Store.KV.find "Title" conv.Conversion.kv
with Not_found -> "" in
let replace x = let open Str in
global_replace (regexp "{{archive-title}}") site_title x
@ -57,7 +57,8 @@ let page htm conversion text =
let authors = (Person.Set.to_string text.authors ^ " ") in
let keywords = str_set "keywords" text in
let header =
let time x = {|<time datetime="|} ^ x ^ {|">|} ^ x ^ "</time>" in
let time x = Printf.sprintf {|<time datetime="%s">%s</time>|}
(Date.rfc_string x) (Date.pretty_date x) in
let topic_links x =
let to_linked t a =
let ts = Topic_set.of_string t in
@ -66,7 +67,7 @@ let page htm conversion text =
"<article><header><dl>"
^ opt_kv "Title:" text.title
^ opt_kv "Authors:" authors
^ opt_kv "Date: " (time (Date.(pretty_date @@ listing text.date)))
^ opt_kv "Date: " (time (Date.listing text.date))
^ opt_kv "Series: " (str_set "series" text)
^ opt_kv "Topics: " (topic_links (set "topics" text))
^ opt_kv "Keywords: " keywords
@ -161,24 +162,15 @@ let topic_sub_index conv htm topic_map topic_root metas =
(* ^ {|<a href=".atom" id="feed">|}^ String.capitalize_ascii topic_root ^{| feed </a>|}*)
^ listing_index topic_map [topic_root] "" metas)
open Logarion
let indices htm c =
let file name = Logarion.File_store.file (Filename.concat c.Conversion.dir name) in
let index_name = try Store.KV.find "HTM-index" c.Conversion.kv with Not_found -> "index.html" in
let title = try Store.KV.find "Title" c.Conversion.kv with Not_found -> "" in
if index_name <> "" then
file index_name (topic_main_index c htm c.topic_roots c.texts);
let index_name = try Logarion.Store.KV.find "HTM-index" c.Conversion.kv with Not_found -> "index.html" in
if index_name <> "" then file index_name (topic_main_index c htm c.topic_roots c.texts);
file "index.date.htm" (date_index c htm c.texts);
List.iter
(fun root -> file ("index." ^ root ^ ".htm") (topic_sub_index c htm c.topics root c.texts))
c.topic_roots;
c.topic_roots
let base_url = try
let locs = Store.KV.find "Locations" c.kv in
let _i = Str.(search_forward (regexp "https?://[^;]*") locs 0) in
Str.(matched_string locs)
with Not_found -> prerr_endline "Missing location for HTTP(S)"; "" in
file "feed.atom" (Atom.feed title c.id base_url "text/html" c.texts)
let converter kv =
let htm = init kv in
Conversion.{ ext; page = Some (page htm); indices = Some (indices htm) }