Skip to content

Dan gitbook fix 1 #1247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 20, 2023
Merged

Dan gitbook fix 1 #1247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
712a119
build cms files on build, add gitbook features
chillenberger Dec 19, 2023
b728403
remove empty test
chillenberger Dec 19, 2023
87b02b1
add tests remove parse html on start
chillenberger Dec 20, 2023
0cc2649
Merge branch 'master' into Dan-gitbook-fix-1
chillenberger Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 174 additions & 92 deletions pgml-dashboard/src/api/cms.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,94 @@ use crate::{
utils::config,
};

use serde::{Deserialize, Serialize};

lazy_static! {
static ref BLOG: Collection = Collection::new("Blog", true);
static ref CAREERS: Collection = Collection::new("Careers", true);
static ref DOCS: Collection = Collection::new("Docs", false);
}

#[derive(Debug, Serialize, Deserialize)]
pub struct Document {
/// The absolute path on disk
pub path: PathBuf,
pub description: Option<String>,
pub image: Option<String>,
pub title: String,
pub toc_links: Vec<TocLink>,
pub html: String,
}

impl Document {
pub async fn from_path(path: &PathBuf) -> anyhow::Result<Document> {
let contents = tokio::fs::read_to_string(&path).await?;

let parts = contents.split("---").collect::<Vec<&str>>();

let (description, contents) = if parts.len() > 1 {
match YamlLoader::load_from_str(parts[1]) {
Ok(meta) => {
if meta.len() == 0 || meta[0].as_hash().is_none() {
(None, contents)
} else {
let description: Option<String> = match meta[0]["description"].is_badvalue()
{
true => None,
false => Some(meta[0]["description"].as_str().unwrap().to_string()),
};
(description, parts[2..].join("---").to_string())
}
}
Err(_) => (None, contents),
}
} else {
(None, contents)
};

// Parse Markdown
let arena = Arena::new();
let spaced_contents = crate::utils::markdown::gitbook_preprocess(&contents);
let root = parse_document(&arena, &spaced_contents, &crate::utils::markdown::options());

// Title of the document is the first (and typically only) <h1>
let title = crate::utils::markdown::get_title(root).unwrap();
let toc_links = crate::utils::markdown::get_toc(root).unwrap();
let image = crate::utils::markdown::get_image(root);
crate::utils::markdown::wrap_tables(root, &arena).unwrap();

// MkDocs, gitbook syntax support, e.g. tabs, notes, alerts, etc.
crate::utils::markdown::mkdocs(root, &arena).unwrap();

// Style headings like we like them
let mut plugins = ComrakPlugins::default();
let headings = crate::utils::markdown::MarkdownHeadings::new();
plugins.render.heading_adapter = Some(&headings);
plugins.render.codefence_syntax_highlighter =
Some(&crate::utils::markdown::SyntaxHighlighter {});

let mut html = vec![];
format_html_with_plugins(
root,
&crate::utils::markdown::options(),
&mut html,
&plugins,
)
.unwrap();
let html = String::from_utf8(html).unwrap();

let document = Document {
path: path.to_owned(),
description,
image,
title,
toc_links,
html,
};
Ok(document)
}
}

/// A Gitbook collection of documents
#[derive(Default)]
struct Collection {
Expand Down Expand Up @@ -62,6 +144,7 @@ impl Collection {

pub async fn get_asset(&self, path: &str) -> Option<NamedFile> {
info!("get_asset: {} {path}", self.name);

NamedFile::open(self.asset_dir.join(path)).await.ok()
}

Expand All @@ -79,7 +162,7 @@ impl Collection {

let path = self.root_dir.join(format!("{}.md", path.to_string_lossy()));

self.render(&path, cluster, self).await
self.render(&path, cluster).await
}

/// Create an index of the Collection based on the SUMMARY.md from Gitbook.
Expand Down Expand Up @@ -173,109 +256,35 @@ impl Collection {
Ok(links)
}

async fn render<'a>(
&self,
path: &'a PathBuf,
cluster: &Cluster,
collection: &Collection,
) -> Result<ResponseOk, Status> {
// Read to string0
let contents = match tokio::fs::read_to_string(&path).await {
Ok(contents) => {
info!("loading markdown file: '{:?}", path);
contents
}
Err(err) => {
warn!("Error parsing markdown file: '{:?}' {:?}", path, err);
return Err(Status::NotFound);
}
};
let parts = contents.split("---").collect::<Vec<&str>>();
let (description, contents) = if parts.len() > 1 {
match YamlLoader::load_from_str(parts[1]) {
Ok(meta) => {
if !meta.is_empty() {
let meta = meta[0].clone();
if meta.as_hash().is_none() {
(None, contents.to_string())
} else {
let description: Option<String> = match meta["description"]
.is_badvalue()
{
true => None,
false => Some(meta["description"].as_str().unwrap().to_string()),
};

(description, parts[2..].join("---").to_string())
}
} else {
(None, contents.to_string())
}
}
Err(_) => (None, contents.to_string()),
}
} else {
(None, contents.to_string())
};

// Parse Markdown
let arena = Arena::new();
let root = parse_document(&arena, &contents, &crate::utils::markdown::options());

// Title of the document is the first (and typically only) <h1>
let title = crate::utils::markdown::get_title(root).unwrap();
let toc_links = crate::utils::markdown::get_toc(root).unwrap();
let image = crate::utils::markdown::get_image(root);
crate::utils::markdown::wrap_tables(root, &arena).unwrap();

// MkDocs syntax support, e.g. tabs, notes, alerts, etc.
crate::utils::markdown::mkdocs(root, &arena).unwrap();

// Style headings like we like them
let mut plugins = ComrakPlugins::default();
let headings = crate::utils::markdown::MarkdownHeadings::new();
plugins.render.heading_adapter = Some(&headings);
plugins.render.codefence_syntax_highlighter =
Some(&crate::utils::markdown::SyntaxHighlighter {});

// Render
let mut html = vec![];
format_html_with_plugins(
root,
&crate::utils::markdown::options(),
&mut html,
&plugins,
)
.unwrap();
let html = String::from_utf8(html).unwrap();

// Handle navigation
// TODO organize this functionality in the collection to cleanup
let index: Vec<IndexLink> = self
.index
// Sets specified index as currently viewed.
fn open_index(&self, path: PathBuf) -> Vec<IndexLink> {
self.index
.clone()
.iter_mut()
.map(|nav_link| {
let mut nav_link = nav_link.clone();
nav_link.should_open(path);
nav_link.should_open(&path);
nav_link
})
.collect();
.collect()
}

// renders document in layout
async fn render<'a>(&self, path: &'a PathBuf, cluster: &Cluster) -> Result<ResponseOk, Status> {
let doc = Document::from_path(&path).await.unwrap();
let index = self.open_index(doc.path);

let user = if cluster.context.user.is_anonymous() {
None
} else {
Some(cluster.context.user.clone())
};

let mut layout = crate::templates::Layout::new(&title, Some(cluster));
if let Some(image) = image {
// translate relative url into absolute for head social sharing
let parts = image.split(".gitbook/assets/").collect::<Vec<&str>>();
let image_path = collection.url_root.join(".gitbook/assets").join(parts[1]);
layout.image(config::asset_url(image_path.to_string_lossy()).as_ref());
let mut layout = crate::templates::Layout::new(&doc.title, Some(cluster));
if let Some(image) = doc.image {
layout.image(&config::asset_url(image.into()));
}
if let Some(description) = &description {
if let Some(description) = &doc.description {
layout.description(description);
}
if let Some(user) = &user {
Expand All @@ -285,11 +294,11 @@ impl Collection {
let layout = layout
.nav_title(&self.name)
.nav_links(&index)
.toc_links(&toc_links)
.toc_links(&doc.toc_links)
.footer(cluster.context.marketing_footer.to_string());

Ok(ResponseOk(
layout.render(crate::templates::Article { content: html }),
layout.render(crate::templates::Article { content: doc.html }),
))
}
}
Expand Down Expand Up @@ -365,6 +374,10 @@ pub fn routes() -> Vec<Route> {
mod test {
use super::*;
use crate::utils::markdown::{options, MarkdownHeadings, SyntaxHighlighter};
use regex::Regex;
use rocket::http::{ContentType, Cookie, Status};
use rocket::local::asynchronous::Client;
use rocket::{Build, Rocket};

#[test]
fn test_syntax_highlighting() {
Expand Down Expand Up @@ -452,4 +465,73 @@ This is the end of the markdown
!html.contains(r#"<div class="overflow-auto w-100">"#) || !html.contains(r#"</div>"#)
);
}

async fn rocket() -> Rocket<Build> {
dotenv::dotenv().ok();
rocket::build()
.manage(crate::utils::markdown::SearchIndex::open().unwrap())
.mount("/", crate::api::cms::routes())
}

fn gitbook_test(html: String) -> Option<String> {
// all gitbook expresions should be removed, this catches {% %} nonsupported expressions.
let re = Regex::new(r"[{][%][^{]*[%][}]").unwrap();
let rsp = re.find(&html);
if rsp.is_some() {
return Some(rsp.unwrap().as_str().to_string());
}

// gitbook TeX block not supported yet
let re = Regex::new(r"(\$\$).*(\$\$)").unwrap();
let rsp = re.find(&html);
if rsp.is_some() {
return Some(rsp.unwrap().as_str().to_string());
}

None
}

// Ensure blogs render and there are no unparsed gitbook components.
#[sqlx::test]
async fn render_blogs_test() {
let client = Client::tracked(rocket().await).await.unwrap();
let blog: Collection = Collection::new("Blog", true);

for path in blog.index {
let req = client.get(path.clone().href);
let rsp = req.dispatch().await;
let body = rsp.into_string().await.unwrap();

let test = gitbook_test(body);

assert!(
test.is_none(),
"bad html parse in {:?}. This feature is not supported {:?}",
path.href,
test.unwrap()
)
}
}

// Ensure Docs render and ther are no unparsed gitbook compnents.
#[sqlx::test]
async fn render_guides_test() {
let client = Client::tracked(rocket().await).await.unwrap();
let docs: Collection = Collection::new("Docs", true);

for path in docs.index {
let req = client.get(path.clone().href);
let rsp = req.dispatch().await;
let body = rsp.into_string().await.unwrap();

let test = gitbook_test(body);

assert!(
test.is_none(),
"bad html parse in {:?}. This feature is not supported {:?}",
path.href,
test.unwrap()
)
}
}
}
Loading