teamcraft/
textindex.rs

1//! Text extraction for the full-text search over file contents.
2//!
3//! * STEP (`.step`, `.stp`, `.p21`) and IGES (`.iges`, `.igs`): the quoted strings of
4//!   the file – product names, descriptions, authors, organisation, layer and
5//!   entity names – without the numeric geometry.
6//! * Plain text formats (txt, md, csv, json, xml, yaml, log, Teamcenter PLM XML,
7//!   OpenSCAD, motor thrust curves, …): the text itself, line by line, with its
8//!   decimal numbers – a text or CSV report reads like a PDF one (charset: byte
9//!   order mark, (mostly valid) UTF-8, else Windows-1252).
10//! * Geometry as text (ASCII STL, OBJ, KiCad boards and symbols): the words
11//!   without the decimal numbers (coordinates are noise in a search).
12//! * PDF: the text layer, read in an isolated process (see `pdftext`); scanned
13//!   PDFs have none.
14//! * HTML (`.html`, `.htm`, the reports of FEA/CFD tools): the readable text with
15//!   table rows as lines, read in the same isolated process (see `htmltext`).
16//! * Everything else (binary CAD, images): nothing (not indexed yet).
17//!
18//! At most `MAX_READ` bytes are read and `MAX_TEXT` characters are kept.
19#![deny(
20    clippy::unwrap_used,
21    clippy::expect_used,
22    clippy::indexing_slicing,
23    clippy::panic,
24    clippy::unreachable
25)]
26
27use std::path::Path;
28
29use tokio::io::AsyncReadExt;
30
31/// Bytes read from the beginning of a file.
32const MAX_READ: u64 = 4 * 1024 * 1024;
33/// Characters kept for indexing.
34const MAX_TEXT: usize = 200_000;
35
36#[derive(Clone, Copy, Debug, PartialEq, Eq)]
37enum Kind {
38    QuotedStrings,
39    /// The text as it is (lines and decimal numbers kept).
40    PlainText,
41    /// Text full of coordinates: words only, decimal numbers dropped.
42    Geometry,
43    /// PDF: read in an isolated process (`pdftext`), only by [`extract_file`].
44    Pdf,
45    /// HTML: read in an isolated process (`htmltext`), only by [`extract_file`].
46    Html,
47    None,
48}
49
50fn kind_of(filename: &str) -> Kind {
51    let ext = filename
52        .rsplit_once('.')
53        .map(|(_, e)| e.to_ascii_lowercase())
54        .unwrap_or_default();
55    match ext.as_str() {
56        "pdf" => Kind::Pdf,
57        "html" | "htm" => Kind::Html,
58        "stl" | "obj" | "kicad_sch" | "kicad_pcb" | "kicad_sym" | "kicad_mod" => Kind::Geometry,
59        "step" | "stp" | "p21" | "iges" | "igs" => Kind::QuotedStrings,
60        "txt" | "md" | "csv" | "tsv" | "json" | "xml" | "yaml" | "yml" | "toml" | "ini" | "cfg"
61        | "log" | "kicad_pro" | "net" | "sch" | "plmxml" | "scad" | "eng" | "rse" | "tex"
62        | "rst" | "py" | "rs" | "c" | "h" | "cpp" | "hpp" | "m"
63        // solver logs, report files, print output and scripts (Fluent,
64        // STAR-CCM+, Nastran)
65        | "out" | "trn" | "f06" | "f04" | "pch" | "jou" | "java" => Kind::PlainText,
66        _ => Kind::None,
67    }
68}
69
70/// Whether the content of such a file is indexed for the full-text search.
71pub fn indexes(filename: &str) -> bool {
72    kind_of(filename) != Kind::None
73}
74
75/// Extracts searchable text from raw file content.
76pub fn extract(filename: &str, content: &[u8]) -> String {
77    let text = match kind_of(filename) {
78        Kind::None | Kind::Pdf | Kind::Html => return String::new(),
79        // A text extension with binary content (binary STL): nothing to index.
80        Kind::PlainText | Kind::Geometry
81            if content.contains(&0) && encoding_rs::Encoding::for_bom(content).is_none() =>
82        {
83            return String::new()
84        }
85        Kind::PlainText => return plain_text(&crate::htmltext::decode(content, false)),
86        // Drop decimal numbers (e.g. the facet coordinates of ASCII STL files);
87        // integers like "912" in "DIN 912" stay searchable.
88        Kind::Geometry => String::from_utf8_lossy(content)
89            .split_whitespace()
90            .filter(|w| !is_decimal_number(w))
91            .collect::<Vec<_>>()
92            .join(" "),
93        // Quoted STEP/IGES strings are names – keep them as they are ("1.4404").
94        Kind::QuotedStrings => quoted_strings(&String::from_utf8_lossy(content)),
95    };
96    // PostgreSQL TEXT cannot hold NUL; other control characters are noise.
97    text.chars()
98        .map(|c| if c.is_control() { ' ' } else { c })
99        .take(MAX_TEXT)
100        .collect()
101}
102
103/// A text file as text: lines kept (rows of a table or report stay rows),
104/// control characters except tabs become spaces (PostgreSQL TEXT cannot hold
105/// NUL), trailing blanks and runs of blank lines go, at most [`MAX_TEXT`]
106/// characters.
107fn plain_text(text: &str) -> String {
108    let mut out = String::new();
109    let mut chars = 0usize;
110    let mut blank = 0;
111    for line in text.lines() {
112        let line: String = line
113            .chars()
114            .map(|c| if c.is_control() && c != '\t' { ' ' } else { c })
115            .collect();
116        let line = line.trim_end();
117        if line.is_empty() {
118            blank += 1;
119            if blank > 1 {
120                continue;
121            }
122        } else {
123            blank = 0;
124        }
125        out.push_str(line);
126        out.push('\n');
127        chars += line.chars().count() + 1;
128        if chars > MAX_TEXT {
129            break;
130        }
131    }
132    out.trim_matches('\n').chars().take(MAX_TEXT).collect()
133}
134
135/// `1.5`, `-0.25`, `1.000000e+00` – but not `912` or `M6`.
136fn is_decimal_number(word: &str) -> bool {
137    word.contains('.')
138        && word.chars().any(|c| c.is_ascii_digit())
139        && word
140            .chars()
141            .all(|c| c.is_ascii_digit() || matches!(c, '.' | '-' | '+' | 'e' | 'E'))
142}
143
144/// Reads (the beginning of) a stored file and extracts its text. A large
145/// plain-text file (solver log, print output) keeps its end as well: the
146/// text of the beginning, the marker [`TAIL_MARK`] and the text of the last
147/// `TAIL_READ` (64 KiB) bytes – where a solver writes its status and final
148/// values. The end is only added when the beginning was text and the end is
149/// text in the same encoding (no NUL, no UTF-16 byte order mark at the start).
150pub async fn extract_file(path: &Path, filename: &str) -> String {
151    if kind_of(filename) == Kind::None {
152        return String::new();
153    }
154    let Ok(mut file) = tokio::fs::File::open(path).await else {
155        return String::new();
156    };
157    match kind_of(filename) {
158        Kind::Pdf => return crate::pdftext::extract_file(file).await,
159        Kind::Html => return crate::htmltext::extract_file(file).await,
160        _ => {}
161    }
162    let len = file.metadata().await.map(|m| m.len()).unwrap_or(0);
163    let mut content = Vec::new();
164    if (&mut file)
165        .take(MAX_READ)
166        .read_to_end(&mut content)
167        .await
168        .is_err()
169    {
170        return String::new();
171    }
172    let tail = if kind_of(filename) == Kind::PlainText && len > 2 * TAIL_READ {
173        read_tail(&mut file, TAIL_READ).await
174    } else {
175        Vec::new()
176    };
177    // CPU work on up to MAX_READ bytes: off the async worker threads, so
178    // other requests are not held up by a large upload
179    let filename = filename.to_string();
180    tokio::task::spawn_blocking(move || {
181        let text = extract(&filename, &content);
182        let cut = len > MAX_READ || text.chars().count() >= MAX_TEXT;
183        if cut && tail_usable(&content, &text, &tail) {
184            with_tail(text, &tail)
185        } else {
186            text
187        }
188    })
189    .await
190    .unwrap_or_default()
191}
192
193/// Bytes read from the end of a large plain-text file.
194const TAIL_READ: u64 = 64 * 1024;
195
196/// Whether the end of a file may follow the text of its beginning: the
197/// beginning gave text (a binary file gives none), the end holds no NUL
198/// (binary data, zero padding, UTF-16), and the beginning has no byte order
199/// mark other than UTF-8's – the end has no mark of its own and would be
200/// decoded in another encoding.
201fn tail_usable(head: &[u8], head_text: &str, tail: &[u8]) -> bool {
202    let same_encoding = encoding_rs::Encoding::for_bom(head)
203        .is_none_or(|(encoding, _)| encoding == encoding_rs::UTF_8);
204    !tail.is_empty() && !head_text.is_empty() && !tail.contains(&0) && same_encoding
205}
206
207/// Between the beginning and the end of a file whose middle was left out.
208pub const TAIL_MARK: &str = "\n[… middle of the file left out …]\n";
209
210/// The text of a file's beginning followed by [`TAIL_MARK`] and the text of
211/// its end (`tail` starts at a line), together at most [`MAX_TEXT`]
212/// characters: the end keeps at most a quarter of them.
213fn with_tail(head_text: String, tail: &[u8]) -> String {
214    let tail_text = plain_text(&crate::htmltext::decode(tail, false));
215    let tail_chars = tail_text.chars().count();
216    let keep = MAX_TEXT / 4;
217    let tail_text: String = tail_text
218        .chars()
219        .skip(tail_chars.saturating_sub(keep))
220        .collect();
221    if tail_text.is_empty() {
222        return head_text;
223    }
224    let room = MAX_TEXT
225        .saturating_sub(tail_text.chars().count())
226        .saturating_sub(TAIL_MARK.chars().count());
227    let mut out: String = head_text.chars().take(room).collect();
228    out.push_str(TAIL_MARK);
229    out.push_str(&tail_text);
230    out
231}
232
233/// The last `max` bytes of an open file, starting at a line: the first, cut
234/// line is dropped (unless the whole file fits). Never reads more than `max`
235/// bytes, whatever the file's size; empty on any error. It takes the file
236/// the caller already opened (seeking it), so a file is opened once.
237pub async fn read_tail(file: &mut tokio::fs::File, max: u64) -> Vec<u8> {
238    use tokio::io::AsyncSeekExt;
239    let Ok(len) = file.metadata().await.map(|m| m.len()) else {
240        return Vec::new();
241    };
242    let start = len.saturating_sub(max);
243    if file.seek(std::io::SeekFrom::Start(start)).await.is_err() {
244        return Vec::new();
245    }
246    let mut tail = Vec::new();
247    if file.take(max).read_to_end(&mut tail).await.is_err() {
248        return Vec::new();
249    }
250    if start == 0 {
251        return tail;
252    }
253    from_next_line(tail)
254}
255
256/// The bytes after the first line break (a cut line is no line); nothing
257/// when there is no line break at all.
258fn from_next_line(bytes: Vec<u8>) -> Vec<u8> {
259    match bytes.iter().position(|b| *b == b'\n') {
260        Some(at) => bytes.get(at + 1..).unwrap_or_default().to_vec(),
261        None => Vec::new(),
262    }
263}
264
265/// The contents of all `'…'` strings (STEP uses `''` for a quote inside a string).
266fn quoted_strings(text: &str) -> String {
267    let mut out = String::new();
268    let mut chars = text.chars().peekable();
269    while let Some(c) = chars.next() {
270        if c != '\'' {
271            continue;
272        }
273        let mut current = String::new();
274        loop {
275            match chars.next() {
276                Some('\'') if chars.peek() == Some(&'\'') => {
277                    chars.next();
278                    current.push('\'');
279                }
280                Some('\'') | None => break,
281                Some(ch) => current.push(ch),
282            }
283        }
284        let current = current.trim();
285        if !current.is_empty() && current.chars().any(char::is_alphabetic) {
286            if !out.is_empty() {
287                out.push(' ');
288            }
289            out.push_str(current);
290        }
291    }
292    out
293}
294
295/// Builds a prefix `tsquery` (`flan:* & m6:*`) from free text, or `None` if
296/// the input contains no searchable word.
297pub fn prefix_query(input: &str) -> Option<String> {
298    let terms: Vec<String> = input
299        .split(|c: char| !c.is_alphanumeric())
300        .filter(|t| !t.is_empty())
301        .take(12)
302        .map(|t| format!("{}:*", t.to_lowercase()))
303        .collect();
304    (!terms.is_empty()).then(|| terms.join(" & "))
305}
306
307#[cfg(test)]
308mod tests {
309    use super::*;
310
311    const STEP: &str = "ISO-10303-21;\nHEADER;\nFILE_DESCRIPTION(('Engine mount bracket'),'2;1');\n\
312        FILE_NAME('nx-eng-001.step','2026-11-03T10:00:00',('Erika'),('Example Lab'),'','','');\nENDSEC;\nDATA;\n\
313        #10=PRODUCT('NX-ENG-001','Engine mount','',(#20));\n#11=CARTESIAN_POINT('',(0.,1.5,2.));\n\
314        #12=PRODUCT('M6','Bolt M6x20 DIN 912','It''s steel',(#20));\nENDSEC;\nEND-ISO-10303-21;";
315
316    #[test]
317    fn step_names_are_extracted() {
318        let text = extract("Mount.STEP", STEP.as_bytes());
319        assert!(text.contains("Engine mount bracket"));
320        assert!(text.contains("Bolt M6x20 DIN 912"));
321        assert!(text.contains("It's steel"), "escaped quotes");
322        assert!(text.contains("Example Lab"));
323        assert!(!text.contains("1.5"), "geometry is not indexed");
324    }
325
326    #[test]
327    fn text_and_binary_files() {
328        assert_eq!(
329            extract("notes.md", b"Thrust 2 kN, DIN 912\r\nburn time 4.2 s"),
330            "Thrust 2 kN, DIN 912\nburn time 4.2 s",
331            "text keeps its lines and decimal numbers"
332        );
333        let stl = b"solid hub\n facet normal 0.0 0.0 1.0\n  outer loop\n   vertex 1.5e+01 -2.25 0.0\n  endloop\n endfacet\nendsolid hub";
334        assert_eq!(
335            extract("hub.stl", stl),
336            "solid hub facet normal outer loop vertex endloop endfacet endsolid hub"
337        );
338        assert!(
339            is_decimal_number("1.000000e+00")
340                && !is_decimal_number("M6")
341                && !is_decimal_number("912")
342        );
343        let mut binary_stl = b"solid exported by CAD".to_vec();
344        binary_stl.resize(84, 0);
345        binary_stl.extend_from_slice(&[0x00, 0x00, 0x80, 0x3f, 0x01, 0x00]);
346        assert_eq!(extract("hub.stl", &binary_stl), "", "binary STL");
347        assert_eq!(
348            extract("part.step", b"#1=PRODUCT('Hub\0\x07 v2','',(#2));"),
349            "Hub   v2",
350            "no NUL or control characters reach the database"
351        );
352        assert_eq!(extract("drawing.pdf", b"%PDF-1.7 binary"), "");
353        assert_eq!(extract("no_extension", b"text"), "");
354    }
355
356    #[test]
357    fn which_files_are_indexed() {
358        for name in [
359            "a.STEP",
360            "a.stp",
361            "a.p21",
362            "a.IGES",
363            "a.igs",
364            "notes.TXT",
365            "bom.csv",
366            "x.kicad_sch",
367            "motor.eng",
368            "part.scad",
369            "data.plmxml",
370            "report.pdf",
371        ] {
372            assert!(indexes(name), "{name}");
373        }
374        for name in ["a.sldprt", "a.png", "README", "a.step.gz"] {
375            assert!(!indexes(name), "{name}");
376        }
377    }
378
379    #[test]
380    fn iges_and_quoted_string_rules() {
381        // Strings without letters (numbers, blanks) are geometry, not names.
382        assert_eq!(
383            extract("a.igs", b"'1.5' '  ' 'Flange DN50' '1.4404 steel' '42'"),
384            "Flange DN50 1.4404 steel"
385        );
386        // An unterminated string at the end is still taken.
387        assert_eq!(extract("a.stp", b"#1=PRODUCT('Hub"), "Hub");
388        // Strings are trimmed.
389        assert_eq!(extract("a.stp", b"('  Bolt  ')"), "Bolt");
390    }
391
392    #[test]
393    fn plain_text_is_the_real_text() {
394        assert_eq!(
395            extract("a.csv", b"M6;912;-0.5;1e5;3.;v1.2"),
396            "M6;912;-0.5;1e5;3.;v1.2"
397        );
398        assert_eq!(
399            extract("a.txt", b"len 12.5 mm -3.0 +1.0e-3 ISO 4762"),
400            "len 12.5 mm -3.0 +1.0e-3 ISO 4762",
401            "a report keeps its numbers"
402        );
403        assert_eq!(
404            extract("summary.tsv", b"case\tdp_pa  \n\n\n\nA\t388.25\x07\n"),
405            "case\tdp_pa\n\nA\t388.25",
406            "tabs stay, blank runs collapse, control characters go"
407        );
408        assert_eq!(extract("a.txt", b""), "");
409        assert_eq!(
410            extract("a.txt", "Düse \u{fffd}".as_bytes()),
411            "Düse \u{fffd}",
412            "non-ASCII text is kept"
413        );
414        assert_eq!(extract("a.log", b"85\xB0C"), "85°C", "Windows-1252");
415        assert_eq!(
416            extract(
417                "solve.log",
418                &["T = 85 °C, 3 µm, Düse\n".as_bytes(), b"\xFF step 12\n"].concat()
419            ),
420            "T = 85 °C, 3 µm, Düse\n\u{fffd} step 12",
421            "UTF-8 with one stray byte stays UTF-8"
422        );
423        let mut utf16 = vec![0xFF, 0xFE];
424        for u in "p 1.5".encode_utf16() {
425            utf16.extend_from_slice(&u.to_le_bytes());
426        }
427        assert_eq!(extract("a.txt", &utf16), "p 1.5", "UTF-16 with BOM");
428        assert_eq!(extract("a.txt", b"a\0b"), "", "binary content");
429    }
430
431    #[test]
432    fn geometry_text_drops_decimal_numbers() {
433        assert_eq!(
434            extract("board.kicad_pcb", b"(at 12.5 -3.0) (layer F.Cu) DIN 912"),
435            "(at -3.0) (layer F.Cu) DIN 912"
436        );
437        assert!(!is_decimal_number("1.2.3a") && is_decimal_number("1.2.3"));
438        assert!(!is_decimal_number("-") && !is_decimal_number("."));
439    }
440
441    #[test]
442    fn html_is_not_read_in_process() {
443        assert_eq!(extract("report.html", b"<p>1.5</p>"), "");
444        assert!(indexes("report.HTM") && indexes("r.html"));
445    }
446
447    #[test]
448    fn extracted_text_is_bounded() {
449        let big = "word ".repeat(MAX_TEXT);
450        assert_eq!(extract("a.txt", big.as_bytes()).chars().count(), MAX_TEXT);
451        let lines = "word\n".repeat(MAX_TEXT);
452        assert_eq!(extract("a.txt", lines.as_bytes()).chars().count(), MAX_TEXT);
453        assert_eq!(extract("a.stl", big.as_bytes()).chars().count(), MAX_TEXT);
454    }
455
456    #[tokio::test]
457    async fn stored_files_are_read() {
458        let path = std::env::temp_dir().join(format!("teamcraft-text-{}", uuid::Uuid::new_v4()));
459        std::fs::write(&path, STEP).unwrap();
460        assert!(extract_file(&path, "m.step").await.contains("Engine mount"));
461        assert_eq!(
462            extract_file(&path, "m.bin").await,
463            "",
464            "not indexed by name"
465        );
466        std::fs::remove_file(&path).ok();
467        assert_eq!(extract_file(&path, "m.step").await, "", "missing file");
468    }
469
470    #[test]
471    fn prefix_queries_are_bounded() {
472        let many = (0..20)
473            .map(|i| format!("w{i}"))
474            .collect::<Vec<_>>()
475            .join(" ");
476        let q = prefix_query(&many).unwrap();
477        assert_eq!(q.matches(":*").count(), 12, "at most twelve terms");
478        assert_eq!(
479            prefix_query("A:B"),
480            Some("a:* & b:*".into()),
481            "no tsquery syntax passes"
482        );
483        assert_eq!(prefix_query(""), None);
484    }
485
486    #[test]
487    fn prefix_queries() {
488        assert_eq!(prefix_query("Flan M6"), Some("flan:* & m6:*".into()));
489        assert_eq!(
490            prefix_query("nx-eng-001"),
491            Some("nx:* & eng:* & 001:*".into())
492        );
493        assert_eq!(prefix_query("  ' & | !  "), None);
494        assert_eq!(prefix_query("Düse"), Some("düse:*".into()));
495    }
496
497    #[test]
498    fn solver_text_files_are_plain_text() {
499        for name in [
500            "run.trn",
501            "x.F06",
502            "x.f04",
503            "x.pch",
504            "run.jou",
505            "Macro.java",
506            "report-file-0.out",
507        ] {
508            assert!(indexes(name), "{name}");
509        }
510        assert_eq!(
511            extract("run.trn", b"  500  1.234e-05\n"),
512            "  500  1.234e-05"
513        );
514        assert_eq!(extract("a.out", b"\x7fELF\x02\x01\x00\x00"), "", "binary");
515    }
516
517    #[test]
518    fn the_end_of_a_large_text_follows_the_beginning() {
519        let head = "a".repeat(MAX_TEXT);
520        let out = with_tail(head, b"Iteration 500\nsolution is converged\n");
521        assert_eq!(out.chars().count(), MAX_TEXT);
522        assert!(out.ends_with(&format!("{TAIL_MARK}Iteration 500\nsolution is converged")));
523        // the end keeps at most a quarter of the text, its last part
524        let long_tail = format!("{}\nlast line", "b".repeat(MAX_TEXT / 2));
525        let out = with_tail("head".into(), long_tail.as_bytes());
526        assert!(out.starts_with("head\n[…"));
527        assert!(out.ends_with("last line"));
528        assert!(out.chars().count() <= MAX_TEXT);
529        // an empty end changes nothing
530        assert_eq!(with_tail("head".into(), b""), "head");
531        assert_eq!(from_next_line(b"cut line\nwhole\n".to_vec()), b"whole\n");
532        assert!(from_next_line(b"no line break".to_vec()).is_empty());
533    }
534
535    #[test]
536    fn the_end_follows_only_a_text_beginning_in_the_same_encoding() {
537        assert!(tail_usable(b"Iteration 1\n", "Iteration 1", b"converged\n"));
538        assert!(
539            tail_usable(b"\xEF\xBB\xBFlog\n", "log", b"converged\n"),
540            "UTF-8 BOM"
541        );
542        // binary file: the beginning gave no text
543        assert!(!tail_usable(b"\x7fELF\x02\x01\x00", "", b"converged\n"));
544        // the end holds binary data or zero padding
545        assert!(!tail_usable(b"log\n", "log", b"end\x00\x00\x00"));
546        // UTF-16: the end has no byte order mark of its own
547        assert!(!tail_usable(b"\xFF\xFEl\x00", "l", b"e\x00n\x00d\x00"));
548        assert!(!tail_usable(b"\xFE\xFF\x00l", "l", b"end"));
549        assert!(!tail_usable(b"log\n", "log", b""));
550    }
551
552    #[tokio::test]
553    async fn read_tail_starts_at_a_line_and_reads_at_most_max() {
554        let dir = std::env::temp_dir().join(format!("tc-tail-{}", std::process::id()));
555        std::fs::create_dir_all(&dir).unwrap();
556        let path = dir.join("run.trn");
557        std::fs::write(&path, b"first line\nsecond line\nthird\n").unwrap();
558        let open = || async { tokio::fs::File::open(&path).await.unwrap() };
559        assert_eq!(read_tail(&mut open().await, 12).await, b"third\n");
560        assert_eq!(
561            read_tail(&mut open().await, 1000).await,
562            b"first line\nsecond line\nthird\n"
563        );
564        // a large file: the text keeps the beginning and the end
565        let mut big = "x 1.5\n".repeat(1_000_000);
566        big.push_str("solution is converged\n");
567        std::fs::write(&path, big.as_bytes()).unwrap();
568        let text = extract_file(&path, "run.trn").await;
569        assert!(text.starts_with("x 1.5\n"));
570        assert!(
571            text.ends_with(&format!("{}solution is converged", "")),
572            "{}",
573            &text[text.len() - 40..]
574        );
575        assert!(text.contains(TAIL_MARK));
576        assert!(text.chars().count() <= MAX_TEXT);
577        std::fs::remove_dir_all(&dir).unwrap();
578    }
579}
teamcraft/textindex.rs

teamcraft/
textindex.rs
