{ "$schema": "folio-library-document/v1", "id": "arxiv-2406.01940v2", "title": "Process-Driven Autoformalization in Lean 4", "variant": "pdf", "pages": 32, "tocSource": "outline", "tocMethod": null, "toc": [ { "level": 1, "title": "Introduction", "number": null, "page": 1, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-000-introduction" }, { "level": 1, "title": "Related Work", "number": null, "page": 3, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-001-related-work" }, { "level": 1, "title": "FormL4: Dataset Construction", "number": null, "page": 3, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-002-forml4-dataset-construction" }, { "level": 2, "title": "Data Source", "number": null, "page": 4, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-004-data-source" }, { "level": 2, "title": "Informalization", "number": null, "page": 4, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-003-informalization" }, { "level": 2, "title": "Curation Process", "number": null, "page": 5, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-005-curation-process" }, { "level": 2, "title": "Comparing FormL4 with Existing Autoformalization Datasets", "number": null, "page": 6, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-006-comparing-forml4-with-existing-autoformalization" }, { "level": 1, "title": "Method: Process-Driven Autoformalization", "number": null, "page": 6, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-007-method-process-driven-autoformalization" }, { "level": 2, "title": "Verification Model", "number": null, "page": 6, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-008-verification-model" }, { "level": 2, "title": "Further Enhancement with Back-Propagated Process Feedback", "number": null, "page": 7, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-009-further-enhancement-with-back-propagated-process" }, { "level": 1, "title": "Experiments", "number": null, "page": 7, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-010-experiments" }, { "level": 2, "title": "Autoformalization Enhancement", "number": null, "page": 7, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-011-autoformalization-enhancement" }, { "level": 3, "title": "Experimental Setup", "number": null, "page": 8, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-012-experimental-setup" }, { "level": 3, "title": "Enhanced Autoformalizer Performance", "number": null, "page": 8, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-013-enhanced-autoformalizer-performance" }, { "level": 2, "title": "Further Enhanced Verifier Performance", "number": null, "page": 9, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-014-further-enhanced-verifier-performance" }, { "level": 2, "title": "Human Evaluation on Autoformalizer Performances", "number": null, "page": 10, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-015-human-evaluation-on-autoformalizer-performances" }, { "level": 1, "title": "Conclusion", "number": null, "page": 11, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-016-conclusion" }, { "level": 1, "title": "Future Directions", "number": null, "page": 19, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-017-future-directions" }, { "level": 1, "title": "More Related Works", "number": null, "page": 19, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-018-more-related-works" }, { "level": 1, "title": "Detailed Decomposition Strategy", "number": null, "page": 20, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-019-detailed-decomposition-strategy" }, { "level": 1, "title": "Prompt for Informalization", "number": null, "page": 21, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-020-prompt-for-informalization" }, { "level": 1, "title": "Human Evaluation: Comparative Model Selection", "number": null, "page": 23, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-021-human-evaluation-comparative-model-selection" }, { "level": 2, "title": "Annotation Protocol for Informalization Model Comparison", "number": null, "page": 23, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-022-annotation-protocol-for-informalization-model-co" }, { "level": 3, "title": "Introduction", "number": null, "page": 23, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-023-introduction" }, { "level": 3, "title": "File Structure", "number": null, "page": 23, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-025-file-structure" }, { "level": 3, "title": "Task", "number": null, "page": 24, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-024-task" }, { "level": 3, "title": "Quality Criteria", "number": null, "page": 24, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-026-quality-criteria" }, { "level": 2, "title": "Annotation Results for Informalization Model Comparison", "number": null, "page": 24, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-027-annotation-results-for-informalization-model-com" }, { "level": 1, "title": "Human Evaluation: PDA Dataset Quality", "number": null, "page": 24, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-028-human-evaluation-pda-dataset-quality" }, { "level": 1, "title": "Case Visualization in Comparision with existing datasets", "number": null, "page": 25, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-029-case-visualization-in-comparision-with-existing" }, { "level": 1, "title": "Human Evaluation: Autoformalization Performance Evaluation", "number": null, "page": 25, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-030-human-evaluation-autoformalization-performance-e" }, { "level": 1, "title": "Experimental Details", "number": null, "page": 25, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-031-experimental-details" }, { "level": 2, "title": "Training Settings", "number": null, "page": 25, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-032-training-settings" }, { "level": 2, "title": "Generation Settings", "number": null, "page": 26, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-033-generation-settings" }, { "level": 2, "title": "Lean 4 Compilation", "number": null, "page": 27, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-034-lean-4-compilation" }, { "level": 1, "title": "Comprehensive Evaluation of the Enhanced Autoformalizer", "number": null, "page": 27, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-035-comprehensive-evaluation-of-the-enhanced-autofor" }, { "level": 1, "title": "Model Selection Process", "number": null, "page": 28, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-036-model-selection-process" }, { "level": 1, "title": "Detailed Dataset Comparison Analysis", "number": null, "page": 28, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-037-detailed-dataset-comparison-analysis" }, { "level": 1, "title": "Analysis for Training and Test Data in FormL4", "number": null, "page": 29, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-038-analysis-for-training-and-test-data-in-forml4" }, { "level": 1, "title": "Autoformalization on Real-World Mathematical Reasonings", "number": null, "page": 29, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-039-autoformalization-on-real-world-mathematical-rea" }, { "level": 1, "title": "Data Construction Details", "number": null, "page": 30, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-040-data-construction-details" }, { "level": 2, "title": "Data Preprocessing", "number": null, "page": 30, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-041-data-preprocessing" }, { "level": 2, "title": "Dataset Split", "number": null, "page": 30, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-042-dataset-split" }, { "level": 2, "title": "Data Postprocessing", "number": null, "page": 31, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-043-data-postprocessing" }, { "level": 1, "title": "Details of Automated Autoformalization Evaluation", "number": null, "page": 31, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-044-details-of-automated-autoformalization-evaluatio" }, { "level": 2, "title": "Prompt", "number": null, "page": 31, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-045-prompt" }, { "level": 2, "title": "Preprocessing and Evaluation", "number": null, "page": 31, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-046-preprocessing-and-evaluation" }, { "level": 2, "title": "Detailed Analyses of Existing LLMs on FormL4", "number": null, "page": 32, "pageLabel": null, "confidence": null, "evidence": null, "source": "outline", "section": "sec-047-detailed-analyses-of-existing-llms-on-forml4" } ], "sections": [ { "id": "sec-front-matter", "number": null, "title": "Front matter", "level": 1, "pageStart": 1, "pageEnd": 1, "labelStart": null, "labelEnd": null, "words": 222, "summary": null, "keywords": [ { "term": "kong university", "heading": false }, { "term": "hong kong", "heading": false }, { "term": "formal languages", "heading": false }, { "term": "process-driven autoformalization", "heading": false }, { "term": "jing", "heading": false }, { "term": "pda", "heading": false }, { "term": "technology", "heading": false }, { "term": "science", "heading": false } ], "extract": "Work in progress PROCESS-DRIVEN AUTOFORMALIZATION IN LEAN 4 Jianqiao Lu1∗, Yingjia Wan2∗, Zhengying Liu3, Yinya Huang4, Jing Xiong1, Chengwu Liu5, Jianhao Shen6, Hui Jin3, Jipeng Zhang8, Haiming Wang7, Zhicheng Yang9, Jing Tang8,9, Zhijiang Guo3† 1The University of Hong Kong 2University of Cambridge 3Huawei Noah’s Ark Lab 4City University of Hong Kong 5Peking University 6Huawei Hisilicon 7Sun Yat-sen University", "extractCut": true }, { "id": "sec-000-introduction", "number": null, "title": "Introduction", "level": 1, "pageStart": 1, "pageEnd": 3, "labelStart": null, "labelEnd": null, "words": 1140, "summary": null, "keywords": [ { "term": "coe min", "heading": false }, { "term": "map min", "heading": false }, { "term": "real numbers", "heading": false }, { "term": "automated assessment", "heading": false }, { "term": "process-driven autoformalization", "heading": false }, { "term": "formal languages", "heading": false }, { "term": "propose process-driven", "heading": false }, { "term": "pda", "heading": false } ], "extract": "Autoformalization is the automatic conversion of natural language mathematics into formal lan- guages (Wang et al., 2018; Szegedy, 2020). It reduces the high cost of formalization and bridges the gap between automated mathematical reasoning research and the vast body of natural language mathematical knowledge (Wu et al., 2022; Jiang et al., 2023c). Recent advancements in large language models (LLMs) showed promising", "extractCut": true }, { "id": "sec-001-related-work", "number": null, "title": "Related Work", "level": 1, "pageStart": 3, "pageEnd": 3, "labelStart": null, "labelEnd": null, "words": 398, "summary": null, "keywords": [ { "term": "machine annotation", "heading": false }, { "term": "recent efforts", "heading": false }, { "term": "osv", "heading": false }, { "term": "wang", "heading": false }, { "term": "psv", "heading": false }, { "term": "llms", "heading": false }, { "term": "utilizes", "heading": false }, { "term": "answer", "heading": false } ], "extract": "Autoformalization with LLMs Autoformalization is the task of automatically converting informal theorems and proofs into machine-verifiable formats (Wang et al., 2018; Szegedy, 2020). Early approaches employed neural machine translation methods to translate texts into the Mizar lan- guage (Wang et al., 2020). Recent advancements in LLMs have opened up new possibilities for autoformalization. Researchers have explored", "extractCut": true }, { "id": "sec-002-forml4-dataset-construction", "number": null, "title": "FormL4: Dataset Construction", "level": 1, "pageStart": 3, "pageEnd": 3, "labelStart": null, "labelEnd": null, "words": 87, "summary": null, "keywords": [ { "term": "informalizing", "heading": false }, { "term": "post-processing", "heading": false }, { "term": "dataset", "heading": true } ], "extract": "The rapid development of Lean 4 (de Moura & Ullrich, 2021) necessitates a benchmark to assess LLMs’ autoformalization capabilities. Existing datasets (Jiang et al., 2023a; Ying et al., 2024a) aims to create benchmarks by informalizing formal theorems from existing libraries. However, they rely on zero-shot instructions to collect natural language statements from GPT-4 without quality checks or rigorous", "extractCut": true }, { "id": "sec-003-informalization", "number": null, "title": "Informalization", "level": 2, "pageStart": 3, "pageEnd": 3, "labelStart": null, "labelEnd": null, "words": 74, "summary": null, "keywords": [ { "term": "forml", "heading": false }, { "term": "assessment", "heading": false }, { "term": "alongside", "heading": false } ], "extract": "ization dataset FORML4 for training and evaluation. FORML4 incorporates proof steps alongside statement translation, leveraging formal proof generation as an auxiliary task. Proof steps could enhance formalized statement quality by providing additional context for model reasoning (Huang et al., 2024b). FORML4 encompasses formal-informal pairs of proof steps along with statements, enabling a comprehensive assessment", "extractCut": true }, { "id": "sec-004-data-source", "number": null, "title": "Data Source", "level": 2, "pageStart": 3, "pageEnd": 4, "labelStart": null, "labelEnd": null, "words": 4, "summary": null, "keywords": [], "extract": "3 Work in progress", "extractCut": false }, { "id": "sec-005-curation-process", "number": null, "title": "Curation Process", "level": 2, "pageStart": 4, "pageEnd": 6, "labelStart": null, "labelEnd": null, "words": 1426, "summary": null, "keywords": [ { "term": "basic test", "heading": false }, { "term": "real test", "heading": false }, { "term": "random test", "heading": false }, { "term": "high-quality informalization", "heading": false }, { "term": "informalization process", "heading": false }, { "term": "informalization success", "heading": false }, { "term": "check appendix", "heading": false }, { "term": "natural language", "heading": false } ], "extract": "existing datasets (Section 3.4). 3.1 DATA SOURCE Statement and Proof Extraction We start by extracting formal statements and proofs from Lean 4 theorems in Mathlib 42, one of the most extensive formal mathematics libraries available. This process is adapted from the implementation of LeanDojo3 (Yang et al., 2023a) to search for and extract theorems from Mathlib 4. However, unlike LeanDojo focuses on extracting", "extractCut": true }, { "id": "sec-006-comparing-forml4-with-existing-autoformalization", "number": null, "title": "Comparing FormL4 with Existing Autoformalization Datasets", "level": 2, "pageStart": 6, "pageEnd": 6, "labelStart": null, "labelEnd": null, "words": 96, "summary": null, "keywords": [ { "term": "datasets", "heading": true }, { "term": "forml", "heading": true }, { "term": "autoformalization", "heading": true }, { "term": "compares", "heading": false } ], "extract": "Table 2 compares FORML4 with existing autoformalization datasets, highlighting its unique features. As the largest dataset designed for iterative, process-driven autoformalization training, FORML4 includes both statements and proofs, employing an LLM-based informalization method that departs from traditional formalization approaches. It ensures high-quality data through rigorous inspection and human verification,", "extractCut": true }, { "id": "sec-007-method-process-driven-autoformalization", "number": null, "title": "Method: Process-Driven Autoformalization", "level": 1, "pageStart": 6, "pageEnd": 6, "labelStart": null, "labelEnd": null, "words": 54, "summary": null, "keywords": [ { "term": "methodology", "heading": false }, { "term": "presents", "heading": false }, { "term": "enhancing", "heading": false } ], "extract": "This section presents our approach to enhancing the autoformalization capabilities of LLMs using process feedback. We establish a baseline by fine-tuning an LLM on the FORML4 training set. Then we further introduce a Process-Supervised Verifier (PSV) that incorporates Lean 4 compiler feedback during training (Section 4.1). Finally, we propose a continuous improvement methodology that", "extractCut": false }, { "id": "sec-008-verification-model", "number": null, "title": "Verification Model", "level": 2, "pageStart": 6, "pageEnd": 7, "labelStart": null, "labelEnd": null, "words": 497, "summary": null, "keywords": [ { "term": "step-level loss", "heading": false }, { "term": "error location", "heading": false }, { "term": "labeling strategy", "heading": false }, { "term": "verifier models", "heading": false }, { "term": "compilation outcome", "heading": false }, { "term": "osv model", "heading": false }, { "term": "log", "heading": false }, { "term": "final compilation", "heading": false } ], "extract": "of the Lean 4 compiler (Section 4.2). 4.1 VERIFICATION MODEL We propose to train the verifier by leveraging the granular, process-level feedback provided by the Lean 4 compiler. This method diverges from previous approaches (Wu et al., 2022) that rely solely on binary compilation outcomes. Instead, we employ a more nuanced strategy that assigns labels to each step in the training data based on the “first error", "extractCut": true }, { "id": "sec-009-further-enhancement-with-back-propagated-process", "number": null, "title": "Further Enhancement with Back-Propagated Process Feedback", "level": 2, "pageStart": 7, "pageEnd": 7, "labelStart": null, "labelEnd": null, "words": 228, "summary": null, "keywords": [ { "term": "process feedback", "heading": true }, { "term": "enhanced autoformalizer", "heading": false }, { "term": "verifier", "heading": false }, { "term": "successful", "heading": false }, { "term": "labels", "heading": false }, { "term": "enhancement", "heading": true }, { "term": "step", "heading": false }, { "term": "outputs", "heading": false } ], "extract": "An iterative refinement strategy is designed to leverage feedback from the Lean 4 compiler to continuously improve both the autoformalizer and verifier. This process comprises three key steps: Step 1: Autoformalizer Improvement The verifier evaluates the autoformalizer’s outputs, assigning labels based on their estimated likelihood of successful compilation. This filtering process ensures that subsequent training", "extractCut": true }, { "id": "sec-010-experiments", "number": null, "title": "Experiments", "level": 1, "pageStart": 7, "pageEnd": 7, "labelStart": null, "labelEnd": null, "words": 1, "summary": null, "keywords": [], "extract": "5.1", "extractCut": false }, { "id": "sec-011-autoformalization-enhancement", "number": null, "title": "Autoformalization Enhancement", "level": 2, "pageStart": 7, "pageEnd": 7, "labelStart": null, "labelEnd": null, "words": 11, "summary": null, "keywords": [], "extract": "This section presents our process-driven autoformalization framework and its experimental results.", "extractCut": false }, { "id": "sec-012-experimental-setup", "number": null, "title": "Experimental Setup", "level": 3, "pageStart": 7, "pageEnd": 8, "labelStart": null, "labelEnd": null, "words": 375, "summary": null, "keywords": [ { "term": "closed-source llms", "heading": false }, { "term": "compiled outputs", "heading": false }, { "term": "experimental", "heading": true }, { "term": "finetuned", "heading": false }, { "term": "selected", "heading": false }, { "term": "test", "heading": false }, { "term": "verifier", "heading": false }, { "term": "openai", "heading": false } ], "extract": "autoformalizer, and conclude with the results of our further enhanced verifier model. 7 Work in progress Table 4: Performance of various LLMs on FORML4 in terms of greedy scores. We include both open- source and closed-source LLMs, as well as models finetuned on FORML4 training data. Reported results indicate the percentage of successfully compiled outputs over all the generated ones (%). Model Test Sets Random Test", "extractCut": true }, { "id": "sec-013-enhanced-autoformalizer-performance", "number": null, "title": "Enhanced Autoformalizer Performance", "level": 3, "pageStart": 8, "pageEnd": 9, "labelStart": null, "labelEnd": null, "words": 333, "summary": null, "keywords": [ { "term": "rft vea", "heading": false }, { "term": "combined rft", "heading": false }, { "term": "autoformalizer", "heading": true }, { "term": "semantic alignment", "heading": false }, { "term": "ensuring semantic", "heading": false }, { "term": "performance", "heading": true }, { "term": "strengths", "heading": false }, { "term": "informal statements", "heading": false } ], "extract": "We compare four autoformalizer models: 1) Baseline Autoformalizer (Baseline) 2) Rejective Sam- pling Fine-tuned (RFT) Autoformalizer (Yuan et al., 2023; Wu et al., 2022) 3) Verifier-Enhanced Autoformalizer (VEA) 4) Combined RFT and Verifier-Enhanced Autoformalizer (RFT+VEA). Results are presented in Table 4, and our analysis reveals three key findings: Effectiveness of Finetuning on FORML4: Even our baseline model,", "extractCut": true }, { "id": "sec-014-further-enhanced-verifier-performance", "number": null, "title": "Further Enhanced Verifier Performance", "level": 2, "pageStart": 9, "pageEnd": 10, "labelStart": null, "labelEnd": null, "words": 417, "summary": null, "keywords": [ { "term": "verifier", "heading": true }, { "term": "rft vea", "heading": false }, { "term": "osv", "heading": false }, { "term": "recall", "heading": false }, { "term": "process-supervision training", "heading": false }, { "term": "verifier models", "heading": false }, { "term": "real dataset", "heading": false }, { "term": "psv", "heading": false } ], "extract": "We further enhance our verifier models using high-quality training data generated by the RFT+VEA autoformalizer. We compare outcome-supervision and process-supervision training methods as discussed in Section 4.1. “PSV+” indicates further fine-tuning under process-supervision, building upon “PSV,” while “OSV+” signifies additional refinement from “OSV” with outcome-supervision. Results for the verifier models", "extractCut": true }, { "id": "sec-015-human-evaluation-on-autoformalizer-performances", "number": null, "title": "Human Evaluation on Autoformalizer Performances", "level": 2, "pageStart": 10, "pageEnd": 11, "labelStart": null, "labelEnd": null, "words": 462, "summary": null, "keywords": [ { "term": "human evaluation", "heading": true }, { "term": "enhanced autoformalizer", "heading": false }, { "term": "validity", "heading": false }, { "term": "autoformalized sample", "heading": false }, { "term": "semantic alignment", "heading": false }, { "term": "pda", "heading": false }, { "term": "dataset split", "heading": false }, { "term": "hypotheses", "heading": false } ], "extract": "To accurately investigate the autoformalization performances of our PDA model in different settings, we conduct an extensive post-hoc human evaluation on the autoformalizers’ output about whether the natural-language statements are successfully translated into formal statements. Goal Human experts give the most accurate evaluations on strict semantic alignment between natural and formal languages, which the", "extractCut": true }, { "id": "sec-016-conclusion", "number": null, "title": "Conclusion", "level": 1, "pageStart": 11, "pageEnd": 19, "labelStart": null, "labelEnd": null, "words": 4307, "summary": null, "keywords": [ { "term": "url arxiv", "heading": false }, { "term": "international conference", "heading": false }, { "term": "url net", "heading": false }, { "term": "processing systems", "heading": false }, { "term": "url https", "heading": false }, { "term": "org abs", "heading": false }, { "term": "information processing", "heading": false }, { "term": "arxiv preprint", "heading": false } ], "extract": "In the current study, we introduce a new benchmark FORML4 specifically designed to assess the autoformalization capabilities of LLMs in Lean 4, and propose a processs-drive autoformalization (PDA) training pipeline with iterative process-level feedback. Unlike the existing dataset focuses on translating questions to statements, FORML4 focuses on tapping each statement’s proof steps to implement a more comprehensive,", "extractCut": true }, { "id": "sec-017-future-directions", "number": null, "title": "Future Directions", "level": 1, "pageStart": 19, "pageEnd": 19, "labelStart": null, "labelEnd": null, "words": 114, "summary": null, "keywords": [ { "term": "future", "heading": true }, { "term": "rft", "heading": false }, { "term": "psv", "heading": false }, { "term": "process-supervision", "heading": false } ], "extract": "Our experiments demonstrate the effectiveness of our process-driven autoformalization framework: 1) The combined RFT+VEA approach leverages the strengths of both rejective sampling and verifier- based filtering, leading to superior autoformalization outcomes. 2) Process-supervised fine-tuning (PSV and PSV+) consistently outperforms outcome-supervised methods, indicating its ability to more effectively leverage", "extractCut": true }, { "id": "sec-018-more-related-works", "number": null, "title": "More Related Works", "level": 1, "pageStart": 19, "pageEnd": 20, "labelStart": null, "labelEnd": null, "words": 856, "summary": null, "keywords": [ { "term": "formal datasets", "heading": false }, { "term": "stands out", "heading": false }, { "term": "reasoning process", "heading": false }, { "term": "formal languages", "heading": false }, { "term": "formal mathematics", "heading": false }, { "term": "problems", "heading": false }, { "term": "prompting", "heading": false }, { "term": "works", "heading": true } ], "extract": "Formal Mathematics Formal languages, such as Isabelle (Wenzel et al., 2008), Lean (de Moura et al., 2015), HOL Light (Harrison, 1996), and Coq (Barras et al., 1997), have become integral tools in modern mathematics verification systems. These interactive theorem provers (ITPs) function as programming languages, allowing users to input statements and proofs in a formal language for automatic correctness verification.", "extractCut": true }, { "id": "sec-019-detailed-decomposition-strategy", "number": null, "title": "Detailed Decomposition Strategy", "level": 1, "pageStart": 20, "pageEnd": 21, "labelStart": null, "labelEnd": null, "words": 296, "summary": null, "keywords": [ { "term": "decomposition strategy", "heading": true }, { "term": "natural-language problem", "heading": false }, { "term": "natural language", "heading": false }, { "term": "formal statement", "heading": false }, { "term": "explaining", "heading": false }, { "term": "subtasks", "heading": false }, { "term": "step", "heading": false }, { "term": "term", "heading": false } ], "extract": "Our decomposition strategy for informalization involves instructing the model to perform the follow- ing subtasks sequentially: 1. Translate the formal statement into a natural-language problem. 20 Work in progress 2. Explain the meaning of each step of the formal proof in natural language, based on the definition of the employed lemma or tactics. 3. Write a step-by-step proof of the problem in natural language", "extractCut": true }, { "id": "sec-020-prompt-for-informalization", "number": null, "title": "Prompt for Informalization", "level": 1, "pageStart": 21, "pageEnd": 23, "labelStart": null, "labelEnd": null, "words": 1286, "summary": null, "keywords": [ { "term": "property holds", "heading": false }, { "term": "lemma states", "heading": false }, { "term": "tactic combines", "heading": false }, { "term": "congruent", "heading": false }, { "term": "natural number", "heading": false }, { "term": "even", "heading": false }, { "term": "natural language", "heading": false }, { "term": "mul", "heading": false } ], "extract": "Below is the few-shot prompt template for querying an LLM to perform formalization. The few-shot examples are carefully curated to ensure the semantical equivalence, logical validity, and readability of natural language translations. Given a statement and its proof written in Lean 4’s syntax, please translate them into the semantically equivalent natural language that a human reader can independently understand", "extractCut": true }, { "id": "sec-021-human-evaluation-comparative-model-selection", "number": null, "title": "Human Evaluation: Comparative Model Selection", "level": 1, "pageStart": 23, "pageEnd": 23, "labelStart": null, "labelEnd": null, "words": 1, "summary": null, "keywords": [], "extract": "E.1", "extractCut": false }, { "id": "sec-022-annotation-protocol-for-informalization-model-co", "number": null, "title": "Annotation Protocol for Informalization Model Comparison", "level": 2, "pageStart": 23, "pageEnd": 23, "labelStart": null, "labelEnd": null, "words": 1, "summary": null, "keywords": [], "extract": "E.1.1", "extractCut": false }, { "id": "sec-023-introduction", "number": null, "title": "Introduction", "level": 3, "pageStart": 23, "pageEnd": 23, "labelStart": null, "labelEnd": null, "words": 14, "summary": null, "keywords": [], "extract": "The protocol provides guidance for evaluating the quality of informalization of two sampled models", "extractCut": false }, { "id": "sec-024-task", "number": null, "title": "Task", "level": 3, "pageStart": 23, "pageEnd": 23, "labelStart": null, "labelEnd": null, "words": 97, "summary": null, "keywords": [ { "term": "natural language", "heading": false }, { "term": "translated", "heading": false }, { "term": "functions", "heading": false } ], "extract": "written in Lean 4 syntax to natural language (i.e., informalization), so that the natural language problem statement and proof can be understood by readers without any Lean 4 knowledge. Given a theorem and proof, the models are prompted to respond following the format below: • Theorem: (the given theorem and proof in Lean 4, to be translated) • Problem: (translated theorem statement in natural language) •", "extractCut": true }, { "id": "sec-025-file-structure", "number": null, "title": "File Structure", "level": 3, "pageStart": 23, "pageEnd": 24, "labelStart": null, "labelEnd": null, "words": 201, "summary": null, "keywords": [ { "term": "model output", "heading": false }, { "term": "json files", "heading": false }, { "term": "problem", "heading": false }, { "term": "extracted", "heading": false }, { "term": "gemini", "heading": false }, { "term": "gpt4o", "heading": false } ], "extract": "You are given two model output .json files (sample size = 10). In the file, each sample contains five items: • \"nl\": (past informalized output. Ignore) • \"formal\": formal statement and proof in Lean 4 (i.e., Theorem) • \"gemini_output\" / gpt4o_output: complete model output • \"nl_problem\": extracted from model output (i.e., Problem) • \"nl_explanation\": extracted from model output (i.e., Explanation) • \"nl_proof\":", "extractCut": true }, { "id": "sec-026-quality-criteria", "number": null, "title": "Quality Criteria", "level": 3, "pageStart": 24, "pageEnd": 24, "labelStart": null, "labelEnd": null, "words": 97, "summary": null, "keywords": [ { "term": "intuitive terms", "heading": false }, { "term": "criteria", "heading": true }, { "term": "ideal", "heading": false } ], "extract": "The ideal informalized output should meet the following criteria: 1. Semantically Equivalent to the Lean 4 Theorem and Proof: (informalization success = T) 2. Intuitive Terms without Lean 4 Functions: Both problem statement and proof use intuitive terms without Lean 4 functions mentioned, proves the intended theorem successfully, and can be independently understood without prior knowledge of Lean 4. (informal proof", "extractCut": true }, { "id": "sec-027-annotation-results-for-informalization-model-com", "number": null, "title": "Annotation Results for Informalization Model Comparison", "level": 2, "pageStart": 24, "pageEnd": 24, "labelStart": null, "labelEnd": null, "words": 45, "summary": null, "keywords": [ { "term": "model preference", "heading": false }, { "term": "informalization success", "heading": false }, { "term": "comparison", "heading": true } ], "extract": "Table 7: Comparison of Model Evaluation in Three Metrics Model Metric Average True Rate Inter-Rater Agreement Gemini Informalization Success 80.0% 80.0% Informal Proof Correctness 85.0% 70.0% Model Preference 60.0% 60.0% GPT4o Informalization Success 72.5% 45.0% Informal Proof Correctness 62.5% 45.0% Model Preference 22.5% 55.0% F", "extractCut": false }, { "id": "sec-028-human-evaluation-pda-dataset-quality", "number": null, "title": "Human Evaluation: PDA Dataset Quality", "level": 1, "pageStart": 24, "pageEnd": 25, "labelStart": null, "labelEnd": null, "words": 299, "summary": null, "keywords": [ { "term": "informalization success", "heading": false }, { "term": "selection stage", "heading": false }, { "term": "basic test", "heading": false }, { "term": "four human", "heading": false }, { "term": "random test", "heading": false }, { "term": "success rate", "heading": false }, { "term": "human experts", "heading": false }, { "term": "model selection", "heading": false } ], "extract": "After obtaining the final dataset, we perform a more extensive manual verification on the informalized dataset, compared to the preliminary one in the model selection stage. Because the core goal of FORML4 is to train and evaluate statement autoformalization, the human verification task only includes annotating the informalization success specific to statement translation. We recruited a different group of four", "extractCut": true }, { "id": "sec-029-case-visualization-in-comparision-with-existing", "number": null, "title": "Case Visualization in Comparision with existing datasets", "level": 1, "pageStart": 25, "pageEnd": 25, "labelStart": null, "labelEnd": null, "words": 293, "summary": null, "keywords": [ { "term": "syntax checking", "heading": false }, { "term": "both statements", "heading": false }, { "term": "existing", "heading": true }, { "term": "mma", "heading": false }, { "term": "contrast", "heading": false }, { "term": "feedback", "heading": false }, { "term": "datasets", "heading": true }, { "term": "reasoning", "heading": false } ], "extract": "In addition to the summarized comparison of dataset features in 2, below we also provide a visualiza- tion comparison through a data example with the same statement in both our FORML4 training set and existing training sets (Jiang et al., 2023a; Ying et al., 2024a). As shown in Table 8, our FORML4 incorporates both the informal statement and its proof as input for our autoformalization process, making it a complete", "extractCut": true }, { "id": "sec-030-human-evaluation-autoformalization-performance-e", "number": null, "title": "Human Evaluation: Autoformalization Performance Evaluation", "level": 1, "pageStart": 25, "pageEnd": 25, "labelStart": null, "labelEnd": null, "words": 36, "summary": null, "keywords": [ { "term": "sample", "heading": false }, { "term": "minutes", "heading": false }, { "term": "cross-evaluate", "heading": false } ], "extract": "EVALUATION The same four annotators for the FORML4 informalization verification task are asked to cross-evaluate 60 autoformalized samples. Each sample is annotated twice. It takes each annotator approximately 5 minutes to complete evaluating a sample. I", "extractCut": false }, { "id": "sec-031-experimental-details", "number": null, "title": "Experimental Details", "level": 1, "pageStart": 25, "pageEnd": 25, "labelStart": null, "labelEnd": null, "words": 1, "summary": null, "keywords": [], "extract": "I.1", "extractCut": false }, { "id": "sec-032-training-settings", "number": null, "title": "Training Settings", "level": 2, "pageStart": 25, "pageEnd": 26, "labelStart": null, "labelEnd": null, "words": 326, "summary": null, "keywords": [ { "term": "training hyperparameters", "heading": false }, { "term": "max length", "heading": false }, { "term": "coe", "heading": false }, { "term": "angle", "heading": false }, { "term": "learning rate", "heading": false }, { "term": "cosine", "heading": false }, { "term": "natural language", "heading": false }, { "term": "global", "heading": false } ], "extract": "Our experiments were conducted in a computing environment equipped with 8 NVIDIA A100 GPUs, each having 40GB of memory. All models underwent fine-tuning in a full-parameter setting. We employed the AdamW optimizer for model training over 2 epochs, with a batch size of 128. The learning rate was set at 5 × 10−6, incorporating a 3% learning rate warmup period. Below, we present a comprehensive overview of the training", "extractCut": true }, { "id": "sec-033-generation-settings", "number": null, "title": "Generation Settings", "level": 2, "pageStart": 26, "pageEnd": 27, "labelStart": null, "labelEnd": null, "words": 119, "summary": null, "keywords": [ { "term": "generation", "heading": true }, { "term": "unbiased", "heading": false }, { "term": "settings", "heading": true }, { "term": "pass", "heading": false } ], "extract": "In this section, we specify the settings used for model generation to ensure reproducibility across all experiments, including baseline models and variations enhanced with verifiers. For the generation of results using the \"greedy\" strategy, we set the temperature parameter to 0.0 and 0.7 for the \"pass@k\" strategy. To present unbiased results for \"pass@k\", we follow the calculation 26 Work in progress method", "extractCut": true }, { "id": "sec-034-lean-4-compilation", "number": null, "title": "Lean 4 Compilation", "level": 2, "pageStart": 27, "pageEnd": 27, "labelStart": null, "labelEnd": null, "words": 298, "summary": null, "keywords": [ { "term": "compilation", "heading": true }, { "term": "versions", "heading": false }, { "term": "minutes", "heading": false }, { "term": "sources", "heading": false }, { "term": "main", "heading": false }, { "term": "time", "heading": false }, { "term": "libraries", "heading": false }, { "term": "tool", "heading": false } ], "extract": "In this section, we outline the specific versions of libraries utilized and the details about the compila- tion process in Lean 4 in our experiments. Lean 4 Compiler: The Lean 4 Compiler is a critical component of the Lean 4 programming language. This tool enables users to craft effective proof automation tactics within the Lean envi- ronment and transform them into optimized C code. The Lean 4 Compiler in our scope", "extractCut": true }, { "id": "sec-035-comprehensive-evaluation-of-the-enhanced-autofor", "number": null, "title": "Comprehensive Evaluation of the Enhanced Autoformalizer", "level": 1, "pageStart": 27, "pageEnd": 28, "labelStart": null, "labelEnd": null, "words": 172, "summary": null, "keywords": [ { "term": "enhanced autoformalizer", "heading": true }, { "term": "rft verifier", "heading": false }, { "term": "rft vea", "heading": false }, { "term": "further refine", "heading": false }, { "term": "comprehensive", "heading": true }, { "term": "comprehensive performance", "heading": false } ], "extract": "We leverage our enhanced autoformalizer to generate high-quality training data, supervised by the Lean 4 compiler, to further refine the verifier model. This process involves the following steps: 1. Data Generation: We employ the RFT+Verifier enhanced autoformalizer to produce samples from the FORML 4, MATH, and GSM 8K training sets. 2. Compilation Testing: Each generated sample undergoes testing via the Lean 4", "extractCut": true }, { "id": "sec-036-model-selection-process", "number": null, "title": "Model Selection Process", "level": 1, "pageStart": 28, "pageEnd": 28, "labelStart": null, "labelEnd": null, "words": 181, "summary": null, "keywords": [ { "term": "selection process", "heading": true }, { "term": "model selection", "heading": true }, { "term": "translated natural-language", "heading": false }, { "term": "gemini-pro", "heading": false }, { "term": "annotators", "heading": false }, { "term": "both models", "heading": false } ], "extract": "Model Selection Process: Our model selection process involved a rigorous comparative evaluation of GPT-4 and Gemini-Pro-1.5. We sampled 10 inputs from the extracted formal theorems and recruited four human annotators to cross-evaluate the informalization outputs of both models. The evaluation was based on three key metrics: 1. Success of statement informalization: Assessing whether the translated natural-language", "extractCut": true }, { "id": "sec-037-detailed-dataset-comparison-analysis", "number": null, "title": "Detailed Dataset Comparison Analysis", "level": 1, "pageStart": 28, "pageEnd": 29, "labelStart": null, "labelEnd": null, "words": 310, "summary": null, "keywords": [ { "term": "mma", "heading": false }, { "term": "process-driven feedback", "heading": false }, { "term": "workbook", "heading": false }, { "term": "datasets", "heading": false }, { "term": "formal language", "heading": false }, { "term": "benchmarking", "heading": false }, { "term": "forml", "heading": false }, { "term": "easier", "heading": false } ], "extract": "Table 2 compares FORML4 with existing autoformalization datasets. Here, we provide a detailed explanation and analysis for each characteristic: Source Language: FORML4 and MMA use formal language as their source, while others use natural language. This approach aligns with empirical findings by Jiang et al. (2023a) suggesting that informalization is generally easier than formalization, potentially leading to", "extractCut": true }, { "id": "sec-038-analysis-for-training-and-test-data-in-forml4", "number": null, "title": "Analysis for Training and Test Data in FormL4", "level": 1, "pageStart": 29, "pageEnd": 29, "labelStart": null, "labelEnd": null, "words": 384, "summary": null, "keywords": [ { "term": "test", "heading": true }, { "term": "training data", "heading": true }, { "term": "real test", "heading": false }, { "term": "test domains", "heading": false }, { "term": "pre-defined lemmas", "heading": false }, { "term": "forml", "heading": true }, { "term": "basic terms", "heading": false }, { "term": "mistral", "heading": false } ], "extract": "To showcase the connection between the training data provided by FORML4 and the test sets, we conduct standard supervised fine-tuning on the Mistral-7B (Jiang et al., 2023b) model using the training data provided by FORML4, with training hyperparameters detailed in Appendix I.1. We compare it with a model trained on 5k sampled training data provided by FORML4. Their autoformalization performance on our three test", "extractCut": true }, { "id": "sec-039-autoformalization-on-real-world-mathematical-rea", "number": null, "title": "Autoformalization on Real-World Mathematical Reasonings", "level": 1, "pageStart": 29, "pageEnd": 30, "labelStart": null, "labelEnd": null, "words": 171, "summary": null, "keywords": [ { "term": "math training", "heading": false }, { "term": "gsm8k", "heading": false }, { "term": "sft model", "heading": false }, { "term": "mistral", "heading": false }, { "term": "autoformalization", "heading": true } ], "extract": "In this section, We list the results of using the SFT model trained in Appendix M, to perform autoformalization based on questions and answers in GSM8K and MATH training sets. The results are presented in the following Table 14. 29 Work in progress Table 14: Comparison of the SFT model’s autoformalization performance, measured by compilation rate (%), on the GSM8K and MATH training sets. Model MATH GSM8K Mistral 0.0", "extractCut": true }, { "id": "sec-040-data-construction-details", "number": null, "title": "Data Construction Details", "level": 1, "pageStart": 30, "pageEnd": 30, "labelStart": null, "labelEnd": null, "words": 1, "summary": null, "keywords": [], "extract": "O.1", "extractCut": false }, { "id": "sec-041-data-preprocessing", "number": null, "title": "Data Preprocessing", "level": 2, "pageStart": 30, "pageEnd": 30, "labelStart": null, "labelEnd": null, "words": 164, "summary": null, "keywords": [ { "term": "names", "heading": false }, { "term": "removed", "heading": false }, { "term": "test", "heading": false }, { "term": "samples", "heading": false }, { "term": "familiar", "heading": false } ], "extract": "Firstly, we retain the “#align” command within the proof, which is used by Mathport6 to connect Lean 3 names to Lean 4 names. This inclusion is intended to facilitate the informalization process for GPT-4 during data construction, as we hypothesize that GPT-4 will better understand the Lean 4 language if there is a connection to the more familiar Lean 3 language. Secondly, all samples with custom Mathlib 4 lemma (as", "extractCut": true }, { "id": "sec-042-dataset-split", "number": null, "title": "Dataset Split", "level": 2, "pageStart": 30, "pageEnd": 31, "labelStart": null, "labelEnd": null, "words": 350, "summary": null, "keywords": [ { "term": "basic test", "heading": false }, { "term": "real test", "heading": false }, { "term": "random test", "heading": false }, { "term": "geometry euclidean", "heading": false }, { "term": "math questions", "heading": false }, { "term": "like mathlib", "heading": false }, { "term": "assesses", "heading": false }, { "term": "geometrical", "heading": false } ], "extract": "The basic test and the real test set are the two added test set data in FORML4 for a more domain- inclusive evaluation of autoformalization. They are collected from distinctive sources compared to the random test set or training data and aimed at assessing nuanced domains of autofomalization capability. Below are detailed descriptions of their features, content, and data creation processes. Basic Test It assesses", "extractCut": true }, { "id": "sec-043-data-postprocessing", "number": null, "title": "Data Postprocessing", "level": 2, "pageStart": 31, "pageEnd": 31, "labelStart": null, "labelEnd": null, "words": 114, "summary": null, "keywords": [ { "term": "length", "heading": false }, { "term": "characters", "heading": false }, { "term": "instances", "heading": false } ], "extract": "We apply a post-filtering process to both the training and test sets to uphold the quality of data examples. The exclusion criteria were as follows: • Instances where the API failed or produced empty content during the informalization stage. • Cases where the length of the natural-language question or answer did not exceed 400 char- acters, or the length of the formalized theorem and proof did not exceed 200", "extractCut": true }, { "id": "sec-044-details-of-automated-autoformalization-evaluatio", "number": null, "title": "Details of Automated Autoformalization Evaluation", "level": 1, "pageStart": 31, "pageEnd": 31, "labelStart": null, "labelEnd": null, "words": 1, "summary": null, "keywords": [], "extract": "P.1", "extractCut": false }, { "id": "sec-045-prompt", "number": null, "title": "Prompt", "level": 2, "pageStart": 31, "pageEnd": 31, "labelStart": null, "labelEnd": null, "words": 61, "summary": null, "keywords": [ { "term": "prompt", "heading": true }, { "term": "template", "heading": false }, { "term": "natural language", "heading": false } ], "extract": "We used a specific instruction prompt for autoformalization with all existing LLMs. The prompt is as follows: Statement and proof in natural language: # Statement: Statement # Proof: proof Translate the statement and proof in natural language to lean4: For the instruction-finetuning model, we used the prompt template and inserted our autoformalization prompt into their template to ensure consistent performance. P.2", "extractCut": false }, { "id": "sec-046-preprocessing-and-evaluation", "number": null, "title": "Preprocessing and Evaluation", "level": 2, "pageStart": 31, "pageEnd": 32, "labelStart": null, "labelEnd": null, "words": 400, "summary": null, "keywords": [ { "term": "pass", "heading": false }, { "term": "greedy", "heading": false }, { "term": "model performance", "heading": false }, { "term": "confidence", "heading": false }, { "term": "unbiased", "heading": false }, { "term": "openai", "heading": false }, { "term": "shao", "heading": false }, { "term": "test", "heading": false } ], "extract": "The model’s response may contain raw text mixed with Lean 4 language, We applied different handling functions to extract the exact Lean 4 language for subsequent compilation. For model responses without any Lean 4 output, we marked them as negative outputs. We employ the metric pass@k to evaluate model performance, defined as the condition where at least one autoformalized instance, comprising both the statement and", "extractCut": true }, { "id": "sec-047-detailed-analyses-of-existing-llms-on-forml4", "number": null, "title": "Detailed Analyses of Existing LLMs on FormL4", "level": 2, "pageStart": 32, "pageEnd": 32, "labelStart": null, "labelEnd": null, "words": 308, "summary": null, "keywords": [ { "term": "greedy score", "heading": false }, { "term": "closed-source llms", "heading": false }, { "term": "open-source llms", "heading": false }, { "term": "closed-source models", "heading": false }, { "term": "scores across", "heading": false }, { "term": "real test", "heading": false }, { "term": "autoformalization tasks", "heading": false }, { "term": "forml", "heading": true } ], "extract": "The emergence of LLMs has fostered advancements in autoformalization tasks, where natural language descriptions are converted into formal, programmable constructs. In this analysis, we examine how various LLMs, benchmarking them across three different tests: Random, Basic, and Real proposed by FORML4. As shown in Table 15, there is a distinguishing performance divide between closed-source and open-source LLMs.", "extractCut": true } ], "checks": {}, "withheld": false, "keywords": [ { "term": "language model", "heading": false }, { "term": "autoformalization model", "heading": false }, { "term": "autoformalization training", "heading": false }, { "term": "forml training", "heading": true }, { "term": "real test", "heading": false }, { "term": "formal language", "heading": false }, { "term": "language compiler", "heading": false }, { "term": "llms autoformalization", "heading": false }, { "term": "statement autoformalization", "heading": false }, { "term": "basic test", "heading": false }, { "term": "verifier model", "heading": false }, { "term": "autoformalization evaluation", "heading": true } ], "links": { "repo": "litlfred/folio-assistant-sci", "branch": "main", "dir": "library/arxiv-2406.01940v2" } }