Daily update. Settled all inner debates on mtgott typing system. About to move mtgott to a separate crate

.gitignore

@@ -1,2 +1,3 @@
 target
 .idea
+src/bin/PRIKOL.rs

src/bin/mtgott_cli.rs

@@ -1,3 +1,41 @@
-fn main() {
+use std::env;
+use std::process;
+use std::path::PathBuf;
+use yyyi_ru::mtgott::charclasses::is_bad_name;
+use yyyi_ru::mtgott::dirsearch::{search_dir};
 
+fn usage() -> ! {
+    eprintln!("Usage: program <path> [-D name value]...");
+    process::exit(1);
+}
+
+fn main() {
+    let mut args = env::args();
+    args.next().unwrap();
+    let path_arg = match args.next() {
+        Some(s) => s,
+        None => usage(),
+    };
+    let mut defines: Vec<(Vec<String>, String)> = Vec::new();
+    while let Some(arg) = args.next() {
+        if arg == "-D" {
+            let name = match args.next() {
+                Some(n) => n,
+                None => usage()
+            };
+            let value = match args.next() {
+                Some(v) => v,
+                None => usage()
+            };
+            defines.push((name.split('.').map(|s| {
+                if is_bad_name(s) {
+                    eprintln!("Bad name: {}", s);
+                    process::exit(1);
+                }
+                String::from(s)
+            }).collect(), value));
+        } else {
+            usage();
+        }
+    }
 }

src/mtgott/charclasses.rs

@@ -31,5 +31,5 @@ pub fn is_bad_name(s: &str) -> bool {
 }
 
 pub fn is_illegal_name(s: &str) -> bool {
-    s == "root" || s == "self" || s == "super"
+    s == "" || s == "root" || s == "self" || s == "super"
 }

src/mtgott/dirsearch.rs

@@ -0,0 +1,142 @@
+use std::{fs, io, fmt};
+use std::fs::{read_dir, metadata, canonicalize};
+use std::path::PathBuf;
+use std::error::Error;
+use super::charclasses::{escape_for_html, is_bad_name};
+use super::runtime::{HigherLevelFunc, Value, DebugState, val_lambda_check_argc};
+use super::parser::{parse_one_file_simplified, parse_one_file_packed};
+use super::lambda_compilation::{plemege_to_value};
+use std::rc::Rc;
+use std::collections::HashMap;
+
+pub fn search_dir_rec_helper<F: Fn(&str) -> bool>(
+    res: &mut Vec<Vec<String>>,
+    fs_path: PathBuf, virtual_path: String, allowed_extensions: &[&str], is_valid_name: &F, recc: u32
+)-> Result<(), Box<dyn Error>> {
+    if recc == 0 { return Err("Recursion limit exceeded".into()); }
+    let fs_path = canonicalize(fs_path)?;
+    for entry in read_dir(&fs_path)? {
+        let entry = entry?;
+        let entry_name = entry.file_name();
+        let entry_name_str = match entry_name.to_str() {Some(x) => x, None => continue };
+        let entry_path = fs_path.join(entry.path());
+        let meta: fs::Metadata = metadata(&entry_path)?;
+        let mut child_virtual_path = virtual_path.clone();
+        if child_virtual_path.len() > 0 { child_virtual_path.push('/') }
+        if meta.is_file() {
+            /* We found our victim! Real fun begins */
+            match allowed_extensions.iter().position(|&e| entry_name_str.ends_with(e)) {
+                Some(i) => {
+                    let without_ext: &str = &entry_name_str[..(entry_name_str.len() - allowed_extensions[i].len())];
+                    if !is_valid_name(without_ext) {continue }
+                    child_virtual_path += without_ext;
+                    res[i].push(child_virtual_path);
+                }
+                None => {}
+            }
+        } else if meta.is_dir() {
+            if !is_valid_name(entry_name_str) { continue }
+            child_virtual_path += entry_name_str;
+            search_dir_rec_helper(res, entry_path, child_virtual_path, allowed_extensions, is_valid_name, recc - 1)?;
+        }
+    }
+    Ok(())
+}
+
+pub fn search_dir<F: Fn(&str) -> bool>(p: &str, allowed_extensions: &[&str], is_valid_name: &F)
+    -> Result<Vec<Vec<String>>, Box<dyn Error>> {
+    let mut res: Vec<Vec<String>> = (0..allowed_extensions.len()).map(|_| Vec::new()).collect();
+    search_dir_rec_helper(&mut res, PathBuf::new(), String::new(), allowed_extensions, is_valid_name, 100)?;
+    Ok(res)
+}
+
+struct MtgottDirContent {
+    mtgott: Vec<String>,
+    imtgott: Vec<String>,
+    plain: Vec<String>
+}
+
+pub fn search_mtgott_dir(p: &str, plain_ext: &str) -> Result<MtgottDirContent, Box<dyn Error>> {
+    let mut all = search_dir(p, &[
+        &(String::from(".mtgott") + plain_ext),
+        &(String::from(".imtgott") + plain_ext),
+        plain_ext
+    ], &is_bad_name)?;
+    Ok(MtgottDirContent{
+        mtgott: std::mem::take(&mut all[0]),
+        imtgott: std::mem::take(&mut all[1]),
+        plain: std::mem::take(&mut all[2]),
+    })
+}
+
+/* Panics if somebody else borrowed root */
+fn add_path_to_root(root: &mut Value, slash_path: &str, obj: Value) -> Result<(), Box<dyn Error>> {
+    let parts: Vec<&str> = slash_path.split('/').collect();
+    let mut cur: &mut Value = root;
+    assert!(parts.len() > 0);
+    for i in 0..parts.len() {
+        match cur {
+            Value::Dict(hashmap) => {
+                cur = Rc::get_mut(hashmap).unwrap().entry(String::from(parts[i])).or_insert(Default::default());
+            }
+            _ => return Err(format!("Overlapping root elements {}", parts[..i].join("/")).into())
+        }
+    }
+    match cur {
+        Value::Int(x) if *x == 0 => {},
+        _ => return Err(format!("Overlapping root elements {}", slash_path).into()),
+    }
+    Ok(())
+}
+
+pub fn get_all_templates(p: &str,  plain_ext: &str) -> Result<Value, Box<dyn Error>> {
+    let source = search_mtgott_dir(p, plain_ext)?;
+    let mut res: Value = Value::Dict(Rc::new(HashMap::new()));
+    for cut_path in source.mtgott {
+        let path = format!("{cut_path}.mtgott{plain_ext}");
+        let text_bytes = fs::read(PathBuf::from(p).join(&path))?;
+        let text = std::str::from_utf8(&text_bytes)?;
+        let plemege = parse_one_file_packed(text)?;
+        let compiled = plemege_to_value(plemege, &path)?;
+        add_path_to_root(&mut res, &cut_path, compiled)?
+    }
+    for cut_path in source.imtgott {
+        let path = format!("{cut_path}.imtgott{plain_ext}");
+        let text_bytes = fs::read(PathBuf::from(p).join(&path))?;
+        let text = std::str::from_utf8(&text_bytes)?;
+        let plemege = parse_one_file_simplified(text)?;
+        let compiled = plemege_to_value(plemege, &path)?;
+        add_path_to_root(&mut res, &cut_path, compiled)?
+    }
+    for cut_path in source.plain {
+        let path = format!("{cut_path}{plain_ext}");
+        let text_bytes = fs::read(PathBuf::from(p).join(&path))?;
+        let text = String::from_utf8(text_bytes)?;
+        add_path_to_root(&mut res, &cut_path, Value::Str(Rc::new(text)))?
+    }
+    Ok(res)
+}
+
+pub fn get_all_templates_plus_builtins(p: &str, plain_ext: &str, sanitize: HigherLevelFunc) -> Result<Value, Box<dyn Error>>{
+    let mut root = get_all_templates(p, plain_ext)?;
+    add_path_to_root(&mut root, "sanitize", Value::Fn(sanitize))?;
+    add_path_to_root(&mut root, "cmd_tag_start", Value::Str(Rc::new(String::from("{%"))))?;
+    add_path_to_root(&mut root, "write_tag_start", Value::Str(Rc::new(String::from("{{"))))?;
+    add_path_to_root(&mut root, "roughinsert_tag_start", Value::Str(Rc::new(String::from("{["))))?;
+    add_path_to_root(&mut root, "magic_block_ending_tag", Value::Str(Rc::new(String::from("{%}"))))?;
+    add_path_to_root(&mut root, "element_ending_tag", Value::Str(Rc::new(String::from("{@}"))))?;
+    Ok(root)
+}
+
+pub fn get_root_html(p: &str) -> Result<Value, Box<dyn Error>>{
+    get_all_templates_plus_builtins(p, ".html", Rc::new(
+        |d_state: &DebugState,  _: &Value, args: &[&Value]| -> Result<Value, Box<dyn Error>> {
+            let _g = d_state.register("In sanitizer".into());
+            val_lambda_check_argc(args, 1)?;
+            match args[0]{
+                Value::Str(s) => Ok(Value::Str(Rc::new(escape_for_html(&s)))),
+                Value::Int(num) => Ok(Value::Str(Rc::new(num.to_string()))),
+                _ => Ok(Value::Str(Rc::new(escape_for_html(&format!("{:?}", args[0])))))
+            }
+        }))
+}

src/mtgott/lambda_compilation.rs

@@ -0,0 +1,27 @@
+use super::runtime::*;
+use super::parser::*;
+use std::error::Error;
+use std::collections::HashMap;
+use std::rc::Rc;
+
+fn cat_vec(arr: Vec<String>) -> String {
+    let total_len: usize = arr.iter().map(|s| { s.len() }).sum();
+    arr.into_iter().fold(String::with_capacity(total_len), |mut acc, p| { acc.push_str(p.as_str()); acc })
+}
+
+/* This function is supposed to compile all elements in one particular file */
+pub fn plemege_to_value(mut x: Plemege, file_path: &str) -> Result<Value, Box<dyn Error>> {
+    match x {
+        Plemege::Package(map) => {
+            let mut new_dict: HashMap<String, Value> = HashMap::new();
+            for (key, thing) in map {
+                new_dict.insert(key, plemege_to_value(thing, file_path)?);
+            }
+            Ok(Value::Dict(Rc::new(new_dict)))
+        },
+        Plemege::Element(el) => {
+            // todo
+            Ok(Value::default())
+        }
+    }
+}

src/mtgott/mod.rs

@@ -1,2 +1,5 @@
-mod charclasses;
+pub mod charclasses;
 pub mod parser;
+pub mod dirsearch;
+pub mod runtime;
+pub mod lambda_compilation;

src/mtgott/parser.rs

@@ -1,5 +1,7 @@
 use std::collections::HashMap;
-use crate::mtgott::charclasses::*;
+use crate::mtgott::charclasses::{is_bad_name, is_digit, is_illegal_name, is_lnspace, is_normal_word_constituent, is_whitespace};
+use std::fmt::{self, Display, Formatter};
+use std::error::Error;
 
 #[derive(Debug, PartialEq, Eq)]
 pub enum Expression {
@@ -90,10 +92,12 @@ pub enum FileParsingErrorKind {
     expected_closing_square_bracket,
     empty_expression_inside_round_brackets,
     empty_expression_inside_square_brackets,
+    unmatched_element_ending_tag,
+    unmatched_magic_block_ending_tag,
+    recursion_limit_exceeded,
 }
 
 use FileParsingErrorKind::*;
-use crate::mtgott::charclasses::{is_bad_name, is_digit, is_illegal_name, is_lnspace, is_normal_word_constituent, is_whitespace};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct FileParsingError {
@@ -108,6 +112,20 @@ impl FileParsingError {
     }
 }
 
+impl Display for FileParsingErrorKind {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl Display for FileParsingError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "FileParsingError: {} at positions {} to {}", self.kind, self.p1, self.p2)
+    }
+}
+
+impl Error for FileParsingError {}
+
 struct Parser<'a> {
     text: &'a str,
     p: usize
@@ -268,6 +286,7 @@ impl<'a> Parser<'a> {
 }
 
 enum BlockEndingTag {
+    EOF,
     /* This is {@} */
     END_ELEMENT,
     /* These tags are command tags */
@@ -388,7 +407,10 @@ impl<'a> Parser<'a> {
         };
 
         loop {
-            if self.is_ahead("{{") {
+            if self.p == self.text.len() {
+                fin_static(self, tp1, &mut res);
+                return finishing_touches(ReasonOfElementEnd{p1: self.p, cmd: BlockEndingTag::EOF}, res);
+            } else if self.is_ahead("{{") {
                 fin_static(self, tp1, &mut res);
                 self.p += 2;
                 let expr: Expression = self.parse_expression(arg_names)?;
@@ -703,7 +725,18 @@ impl<'a> Parser<'a> {
     }
 }
 
-pub fn parse_one_file(text: &str) -> Result<Plemege, FileParsingError> {
+/* Parses a file treating it like a package (where other packages and elements could be located) */
+pub fn parse_one_file_packed(text: &str) -> Result<Plemege, FileParsingError> {
     let mut parser: Parser = Parser{text, p: 0};
     parser.parse_pack_plus_ending(true)
 }
+
+pub fn parse_one_file_simplified(text: &str) -> Result<Plemege, FileParsingError> {
+    let mut parser: Parser = Parser{text, p: 0};
+    let (el, tt): (Element, ReasonOfElementEnd) = parser.parse_element_plus_ending_tag(&Vec::new())?;
+    match tt.cmd {
+        BlockEndingTag::EOF => Ok(Plemege::Element(el)),
+        BlockEndingTag::END_ELEMENT => Err(FileParsingError::new(unmatched_element_ending_tag, tt.p1, parser.p)),
+        _ => Err(FileParsingError::new(unmatched_magic_block_ending_tag, tt.p1, parser.p)),
+    }
+}

src/mtgott/runtime.rs

@@ -0,0 +1,84 @@
+use std::collections::HashMap;
+use std::fmt;
+use std::fmt::Debug;
+use std::ops::{Index, IndexMut};
+use std::error::Error;
+use std::rc::Rc;
+use std::cell::RefCell;
+
+pub struct DebugStateGuard<'a> (
+    &'a RefCell<Vec<String>>
+);
+
+impl<'a> Drop for DebugStateGuard<'a> {
+    fn drop(&mut self) {
+        assert!(matches!(self.0.borrow_mut().pop(), Some(_)));
+    }
+}
+
+pub struct DebugState (
+    RefCell<Vec<String>>
+);
+
+impl<'a> DebugState {
+    pub fn register(&'a self, msg: String) -> DebugStateGuard<'a> {
+        self.0.borrow_mut().push(msg);
+        DebugStateGuard(&self.0)
+    }
+}
+
+pub type HigherLevelFunc = Rc<dyn Fn(&DebugState, &Value, &[&Value]) -> Result<Value, Box<dyn Error>>>;
+
+pub enum Value {
+    Str(Rc<String>),
+    Int(u64),
+    Arr(Rc<Vec<Value>>),
+    Dict(Rc<HashMap<String, Value>>),
+    Fn(HigherLevelFunc),
+}
+
+/* Useful utility to put in every function you see */
+pub fn val_lambda_check_argc(argv: &[&Value], argc_required: usize) -> Result<(), String> {
+    if argv.len() == argc_required {
+        Ok(())
+    } else {
+        Err(format!("Function takes {argc_required} arguments, but {} were given", argv.len()))
+    }
+}
+
+
+impl Default for Value {
+    fn default() -> Self {
+        Value::Int(0)
+    }
+}
+
+impl Debug for Value {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Value::Str(s) => write!(f, "Str({})", s),
+            Value::Int(i) => write!(f, "Int({})", i),
+            Value::Arr(a) => write!(f, "Arr({:?})", a),
+            Value::Dict(d) => {
+                let dict_debug: Vec<String> = d.iter()
+                    .map(|(k, v)| format!("{}: {:?}", k, v))
+                    .collect();
+                write!(f, "Dict({{{}}})", dict_debug.join(", "))
+            }
+            Value::Fn(_) => write!(f, "Fn(<function>)"),
+        }
+    }
+}
+
+impl<'t> PartialEq for Value {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Value::Str(s1), Value::Str(s2)) => s1 == s2,
+            (Value::Int(i1), Value::Int(i2)) => i1 == i2,
+            (Value::Arr(a1), Value::Arr(a2)) => a1 == a2,
+            (Value::Dict(d1), Value::Dict(d2)) => d1 == d2,
+            (Value::Fn(_), Value::Fn(_)) => false,
+            _ => false,
+        }
+    }
+}

tests/parsing_test.rs

@@ -25,9 +25,9 @@ fn test_parse_file_with_all_combinations() {
 
     generate_strings(&mut String::new(), target_length, &alphabet, &mut |s| {
         println!("Parsing {s}");
-        parse_one_file(&s);
+        parse_one_file_packed(&s);
-        parse_one_file((String::from("{% as s e1e %} adasd {%}") + s.as_str()).as_str());
+        parse_one_file_packed((String::from("{% as s e1e %} adasd {%}") + s.as_str()).as_str());
-        parse_one_file((String::from("{% as s e1e %} a{[111 . 2332]]dasd {%} {% as s e1e %} adas {{}}d {%} ") + s.as_str()).as_str());
+        parse_one_file_packed((String::from("{% as s e1e %} a{[111 . 2332]]dasd {%} {% as s e1e %} adas {{}}d {%} ") + s.as_str()).as_str());
     });
 }
 
@@ -42,7 +42,7 @@ macro_rules! string_map {
 
 #[test]
 fn resulting_package() {
-    assert_eq!(parse_one_file("{@ x  @}{@}"),
+    assert_eq!(parse_one_file_packed("{@ x  @}{@}"),
                Ok(Plemege::Package(string_map! {
                    "x" => Plemege::Element(Element{argc: 0, sub_elements: vec![]})
                })))