Feat/ssr for context (#43)

* Update README.md to reflect support for Djot in addition to Markdown

* Change module name for accuracy

* JSON-LD feature integration

* Add sitemap generation and configuration options for improved SEO support

* Apply Prettier format

* 📝 CodeRabbit Chat: Add client and server Gleam tests for parsing, rendering, and caching

* Apply build script's format

* Remove faulty coderabbit test

* ensure proper URL formatting

* Fix parsing errors in sitemap generation function; add descriptions and titles

* Fix sitemap URL formatting in CompleteData structure

* Enhance sitemap handling in request processing; 404 when disabled

* Remove

* Remove news

* remove redundant extraction

* Enhance JSON-LD generation

* Apply Prettier format

* Case insensitive false

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

Author: 3 people

README.md

@@ -1,13 +1,13 @@
 # CynthiaWebsiteEngine-mini
 
 A lightweight website engine emphasising simplicity and ease of use. Optimised for small to medium-sized websites requiring both static and dynamic capabilities.
-Create entire sites out of just a few markdown/html/plaintext files and their json metadata!
+Create entire sites out of just a few djot/markdown/html/plaintext files and their json metadata!
 
 ## Key Features
 
 - 🚀 Simple setup and configuration
 - 🎨 Wide collection of pre-made themes
-- 📝 Markdown and HTML support
+- 📝 Djot and HTML support (MarkDown support with Pandoc)
 - 🔧 Static site generation capabilities
 - 🔌 Extensible plugin architecture based on the Cynthia v3 plugin system (coming in v2)
 
@@ -59,20 +59,20 @@ For detailed information about configuration, theming, and deployment, check out
 
 ```directory
 ./content/
-    index.md
-    index.md.meta.json
+    index.dj
+    index.dj.meta.json
     about.md
     about.md.meta.json
     projects/
         project1.html
         project1.html.meta.json
-        project2.md
-        project2.md.meta.json
+        project2.dj
+        project2.dj.meta.json
     articles/
-        article1.md
-        article1.md.meta.json
-        article2.md
-        article2.md.meta.json
+        article1.dj
+        article1.dj.meta.json
+        article2.dj
+        article2.dj.meta.json
 ./cynthia-mini.toml
 ```
 

cynthia_websites_mini_client/src/cynthia_websites_mini_client/configtype.gleam

@@ -16,6 +16,8 @@ pub type CompleteData {
     server_host: Option(String),
     comment_repo: Option(String),
     git_integration: Bool,
+    crawlable_context: Bool,
+    sitemap: Option(String),
     other_vars: List(#(String, List(String))),
     content: List(Content),
   )
@@ -34,6 +36,8 @@ pub fn encode_complete_data_for_client(complete_data: CompleteData) -> json.Json
     git_integration:,
     other_vars:,
     content:,
+    crawlable_context:,
+    sitemap:,
   ) = complete_data
   json.object([
     #("global_theme", json.string(global_theme)),
@@ -42,6 +46,11 @@ pub fn encode_complete_data_for_client(complete_data: CompleteData) -> json.Json
     #("global_site_name", json.string(global_site_name)),
     #("global_site_description", json.string(global_site_description)),
     #("git_integration", json.bool(git_integration)),
+    #("crawlable_context", json.bool(crawlable_context)),
+    #("sitemap", case sitemap {
+      None -> json.null()
+      Some(value) -> json.string(value)
+    }),
     #("comment_repo", case comment_repo {
       None -> json.null()
       Some(value) -> json.string(value)
@@ -93,6 +102,17 @@ pub fn complete_data_decoder() -> decode.Decoder(CompleteData) {
     |> decode.map(list.fold(_, dict.new(), dict.merge))
   })
 
+  use crawlable_context <- decode.optional_field(
+    "crawlable_context",
+    default_shared_cynthia_config_global_only.crawlable_context,
+    decode.bool,
+  )
+  use sitemap <- decode.optional_field(
+    "sitemap",
+    default_shared_cynthia_config_global_only.sitemap,
+    decode.optional(decode.string),
+  )
+
   let other_vars = dict.to_list(other_vars)
 
   decode.success(CompleteData(
@@ -105,6 +125,8 @@ pub fn complete_data_decoder() -> decode.Decoder(CompleteData) {
     server_host:,
     comment_repo:,
     git_integration:,
+    crawlable_context:,
+    sitemap:,
     other_vars:,
     content:,
   ))
@@ -120,7 +142,18 @@ pub type SharedCynthiaConfigGlobalOnly {
     server_port: Option(Int),
     server_host: Option(String),
     comment_repo: Option(String),
+    /// [True]
+    /// Wether or not to enable git integration for the site.
     git_integration: Bool,
+    /// [False]
+    /// Wether or not to insert json-ld+context into the HTML
+    /// to make the site crawlable by search engines or readable by LLMs.
+    crawlable_context: Bool,
+    /// [True]
+    /// Wether or not to create a sitemap.xml file for the site.
+    /// This is useful for search engines to index the site.
+    /// This is separate from the crawlable_context setting, as no content needs to be rendered or served for the sitemap.xml file.
+    sitemap: Option(String),
     other_vars: List(#(String, List(String))),
   )
 }
@@ -135,6 +168,8 @@ pub const default_shared_cynthia_config_global_only: SharedCynthiaConfigGlobalOn
   server_host: None,
   comment_repo: None,
   git_integration: True,
+  crawlable_context: False,
+  sitemap: Some("https://example.com"),
   other_vars: [],
 )
 
@@ -152,6 +187,8 @@ pub fn merge(
     server_host: orig.server_host,
     comment_repo: orig.comment_repo,
     git_integration: orig.git_integration,
+    crawlable_context: orig.crawlable_context,
+    sitemap: orig.sitemap,
     other_vars: orig.other_vars,
     content:,
   )

cynthia_websites_mini_client/src/cynthia_websites_mini_client/dom.gleam

@@ -37,3 +37,8 @@ pub fn set_hash(hash: String) -> Nil
 /// Get innerhtml of an element
 @external(javascript, "./dom.ts", "get_inner_html")
 pub fn get_inner_html(element: element.Element) -> String
+
+/// jsonify_string
+/// Convert a string to a JSON safe string
+@external(javascript, "./dom.ts", "jsonify_string")
+pub fn jsonify_string(str: String) -> Result(String, Nil)

cynthia_websites_mini_client/src/cynthia_websites_mini_client/dom.ts

@@ -1,3 +1,5 @@
+import { Error, Ok } from "../../prelude";
+
 export function get_color_scheme() {
   // Media queries the preferred color colorscheme
 
@@ -57,3 +59,13 @@ export function destroy_comment_box() {
     }
   }
 }
+
+export function jsonify_string(str: string) {
+  // Convert a string to a JSON object
+  try {
+    return new Ok(JSON.stringify(str));
+  } catch (e) {
+    console.error("Failed to parse JSON string:", e);
+    return new Error(null);
+  }
+}

cynthia_websites_mini_client/src/cynthia_websites_mini_client/shared/jsonld.gleam

@@ -0,0 +1,74 @@
+import cynthia_websites_mini_client
+import cynthia_websites_mini_client/configtype
+import cynthia_websites_mini_client/contenttypes
+import cynthia_websites_mini_client/dom
+import cynthia_websites_mini_client/pottery
+import gleam/list
+import gleam/result
+import gleam/string
+import lustre/element
+
+/// Generates JSON-LD structured data for the website.
+pub fn generate_jsonld(cd: configtype.CompleteData) -> String {
+  let base_jsonld = "{
+    \"@context\": \"https://schema.org\",
+    \"@type\": \"Website\",
+    \"name\": " <> cd.global_site_name
+    |> dom.jsonify_string()
+    |> result.unwrap("Site name is invalid") <> ",
+    \"description\": " <> cd.global_site_description
+    |> dom.jsonify_string()
+    |> result.unwrap("Site description is invalid") <> ",
+    \"generator\": {
+      \"@type\": \"SoftwareApplication\",
+      \"name\": \"CynthiaMini Website Engine\",
+      \"url\": \"https://github.com/CynthiaWebsiteEngine/Mini-docs\",
+      \"version\": \"" <> cynthia_websites_mini_client.version() <> "\"
+    },
+    \"@graph\": ["
+
+  let content_jsonld =
+    cd.content
+    |> list.map(fn(c) {
+      let title = {
+        result.unwrap(dom.jsonify_string(c.title), "Title is invalid")
+      }
+      let description = {
+        pottery.parse_html(c.description, "descr.dj")
+        |> element.to_string()
+        |> dom.jsonify_string()
+        |> result.unwrap("Description is invalid")
+      }
+      let content_type = case c.data {
+        contenttypes.PostData(..) -> "BlogPosting"
+        contenttypes.PageData(..) -> "WebPage"
+      }
+
+      let dates = case c.data {
+        contenttypes.PostData(
+          date_published: published,
+          date_updated: updated,
+          category: _,
+          tags: _,
+        ) -> "\n\"datePublished\": \"" <> published <> "\",
+           \"dateModified\": \"" <> updated <> "\","
+        _ -> ""
+      }
+
+      "{
+        \"@type\": \"" <> content_type <> "\",
+        \"@id\": " <> c.permalink |> dom.jsonify_string() |> result.unwrap("/") <> ",
+        \"headline\": " <> title <> ",
+        \"description\": " <> description <> "," <> dates <> "
+        \"mainEntityOfPage\": {
+          \"@type\": \"WebPage\",
+          \"@id\": " <> c.permalink
+      |> dom.jsonify_string()
+      |> result.unwrap("/") <> "
+        }
+      }"
+    })
+    |> string.join(",\n")
+
+  base_jsonld <> content_jsonld <> "]}"
+}

cynthia_websites_mini_client/src/cynthia_websites_mini_client/shared/sitemap.gleam

@@ -0,0 +1,100 @@
+import cynthia_websites_mini_client/configtype.{type CompleteData}
+import cynthia_websites_mini_client/contenttypes
+import cynthia_websites_mini_client/pottery
+import gleam/list
+import gleam/option
+import gleam/string
+import lustre/attribute.{attribute}
+import lustre/element.{type Element, element}
+
+pub fn generate_sitemap(data: CompleteData) -> option.Option(String) {
+  use base_url: String <- option.then(
+    option.then(data.sitemap, fn(url) {
+      {
+        case url |> string.ends_with("/") {
+          True -> url
+          False -> url <> "/"
+        }
+        <> "#"
+      }
+      |> option.Some
+    }),
+  )
+
+  let all_entries =
+    data.content
+    |> list.map(fn(post) {
+      // We'll get the url, dates, title and description for each post
+      let url = base_url <> post.permalink
+      let lastmod = case post.data {
+        contenttypes.PostData(
+          date_published: published,
+          date_updated: updated,
+          ..,
+        ) ->
+          // If post has an updated date use that, otherwise use published date
+          case updated {
+            "" -> published
+            // Empty string means no update date
+            _ -> updated
+            // Use update date if available
+          }
+        contenttypes.PageData(..) -> ""
+        // Pages don't have dates yet
+      }
+      #(url, lastmod, post.title, post.description)
+    })
+    |> list.map(fn(entry) {
+      // If the entry is the homepage, make it / instead of the base URL
+      let #(url, lastmod, title, desc) = entry
+      let url = case url {
+        "" -> {
+          // If the URL is empty, we assume it's the homepage
+          base_url <> "/"
+        }
+        _ -> url
+      }
+      #(url, lastmod, title, desc)
+    })
+
+  // Create the XML using lustre
+  let urlset =
+    element(
+      "urlset",
+      [attribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")],
+      list.map(all_entries, fn(entry) {
+        let #(url, lastmod, _title, _desc) = entry
+        let mut_elements = [
+          element("loc", [], [element.text(url)]),
+          element("changefreq", [], [element.text("weekly")]),
+          element("priority", [], [element.text("1.0")]),
+          // Description and title are not standard in sitemaps, sadly.
+        // cdata_into_lustre("title", element.text(title)),
+        // cdata_into_lustre("description", pottery.parse_html(desc, "descr.dj")),
+        ]
+        // Only add lastmod if we have a date
+        let elements = case lastmod {
+          "" -> mut_elements
+          date -> [element("lastmod", [], [element.text(date)]), ..mut_elements]
+        }
+        element("url", [], elements)
+      }),
+    )
+
+  // Convert the XML tree to a string
+  option.Some(element.to_readable_string(urlset))
+}
+// fn cdata_into_lustre(
+//   element_tag: String,
+//   inner: Element(a),
+// ) -> element.Element(a) {
+//   // Create a CDATA section in Lustre
+//   // as a string :
+//   // <![CDATA[ ... ]]>
+//   element.unsafe_raw_html(
+//     "",
+//     element_tag,
+//     [],
+//     "<![CDATA[" <> element.to_string(inner) <> "]]>",
+//   )
+// }