Update benchmarks

.codecov.yml

@@ -0,0 +1,10 @@
+coverage:
+  status:
+    patch:
+      default:
+        target: 70%
+        threshold: 2%
+    project:
+      default:
+        target: auto
+        threshold: 10%

.gitignore

@@ -11,6 +11,7 @@ benchmarks/Manifest.toml
 # IDE
 .vscode/
 .idea/
+CLAUDE.md
 
 # OS
 .DS_Store
@@ -19,6 +20,22 @@ Thumbs.db
 # Temporary files
 *.tmp
 *.temp
+*.swp
+*.swo
 
 statprof/
-OptimalBranching.jl/
+OptimalBranching.jl/
+.julia/
+.claude/
+
+# === Build outputs ===
+/benchmarks/artifacts/
+/benchmarks/third-party/**/build/
+/benchmarks/third-party/**/bin/
+/benchmarks/third-party/**/CMakeFiles/
+/benchmarks/third-party/**/Makefile
+/benchmarks/third-party/**/cmake_install.cmake
+
+# === Logs ===
+*.log
+*.out

.gitmodules

@@ -0,0 +1,16 @@
+[submodule "benchmarks/third-party/x-sat"]
+	path = benchmarks/third-party/x-sat
+	url = https://github.com/shaowei-cai-group/X-SAT.git
+	ignore = all
+[submodule "benchmarks/third-party/aiger"]
+	path = benchmarks/third-party/aiger
+	url = https://github.com/arminbiere/aiger.git
+	ignore = all
+[submodule "benchmarks/third-party/abc"]
+	path = benchmarks/third-party/abc
+	url = https://github.com/berkeley-abc/abc.git
+	ignore = all
+[submodule "benchmarks/third-party/CnC"]
+	path = benchmarks/third-party/CnC
+	url = https://github.com/marijnheule/CnC.git
+	ignore = all

Project.toml

@@ -9,6 +9,7 @@ GenericTensorNetworks = "3521c873-ad32-4bb4-b63d-f4f178f42b49"
 OptimalBranchingCore = "c76e7b22-e1d2-40e8-b0f1-f659837787b8"
 ProblemReductions = "899c297d-f7d2-4ebf-8815-a35996def416"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TropicalNumbers = "b3a74e9c-7526-4576-a4eb-79c0d4c32334"
 
 [compat]
@@ -17,6 +18,7 @@ GenericTensorNetworks = "4"
 OptimalBranchingCore = "0.1"
 ProblemReductions = "0.3.5"
 SparseArrays = "1.11.0"
+Statistics = "1.11.1"
 TropicalNumbers = "0.6.2"
 julia = "1"
 

benchmarks/Project.toml

@@ -8,6 +8,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 BooleanInference = "d12a6243-8379-43cb-9459-098119da1519"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
+HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 Primes = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae"
@@ -20,6 +21,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 BenchmarkTools = "1.6.0"
 Dates = "1.11.0"
 Gurobi = "1.7.6"
+HiGHS = "1.19.3"
 JSON3 = "1.14.3"
 JuMP = "1.29.1"
 ProblemReductions = "0.3.5"

benchmarks/examples/factoring_example.jl

@@ -0,0 +1,62 @@
+using Pkg
+Pkg.activate(joinpath(@__DIR__, ".."))
+
+using BooleanInferenceBenchmarks
+using Random
+using Gurobi
+
+const env = Gurobi.Env()
+
+# ----------------------------------------
+# Example 1: Generate datasets
+# ----------------------------------------
+
+println("=" ^80)
+println("Generating Factoring Datasets")
+println("=" ^80)
+
+configs = [
+    # FactoringConfig(10, 10),
+    # FactoringConfig(12, 12),
+    # FactoringConfig(14, 14),
+    # FactoringConfig(16, 16),
+    # FactoringConfig(18, 18),
+    # FactoringConfig(20, 20),
+    FactoringConfig(22, 22),
+    # FactoringConfig(24, 24)
+]
+
+paths = generate_factoring_datasets(configs; per_config=5, include_solution=true, force_regenerate=false)
+
+# ----------------------------------------
+# Example 2: Run benchmark on a single dataset
+# ----------------------------------------
+
+# println("\n" * "=" ^80)
+# println("Running Benchmark")
+# println("=" ^80)
+
+# # result = benchmark_dataset(FactoringProblem, paths[1]; solver=XSATSolver(;yosys_path="/opt/homebrew/bin/yosys"))
+# result = benchmark_dataset(FactoringProblem, paths[1]; solver=IPSolver(Gurobi.Optimizer, env))
+
+# if !isnothing(result)
+#     println("\nResults:")
+#     println("  Dataset: $(result["dataset_path"])")
+#     println("  Instances tested: $(result["instances_tested"])")
+#     println("  Accuracy: $(round(result["accuracy_rate"]*100, digits=1))%")
+#     println("  Median time: $(result["median_time"]) seconds")
+# end
+
+# ----------------------------------------
+# Example 3: Compare solvers
+# ----------------------------------------
+
+println("\n" * "=" ^80)
+println("Comparing Solvers")
+println("=" ^80)
+
+# results = run_solver_comparison(FactoringProblem, paths, solvers=[XSATSolver(;yosys_path="/opt/homebrew/bin/yosys"), BooleanInferenceSolver()])
+# results = run_solver_comparison(FactoringProblem, paths, solvers=[XSATSolver(;yosys_path="/opt/homebrew/bin/yosys")])
+results = run_solver_comparison(FactoringProblem, paths, solvers=[BooleanInferenceSolver()])
+print_solver_comparison_summary(results)
+

benchmarks/src/BooleanInferenceBenchmarks.jl

@@ -2,36 +2,42 @@ module BooleanInferenceBenchmarks
 
 using Random
 using JSON3
+using Primes
+using Dates: now
 using BenchmarkTools
 using BooleanInference
-using JuMP
+using JuMP, HiGHS, Gurobi
+using Statistics: mean, median
+using SHA: bytes2hex, sha256, sha1
 using ProblemReductions
-using Statistics: mean, median, std, quantile
-using Dates: now
-using SHA: bytes2hex, sha256
-
-
-# Core utilities
-include("utils.jl")
+using ProblemReductions: Factoring, CircuitSAT, reduceto, constraints, objectives, AbstractProblem, Symbol
 
-# Abstract types and interfaces
 include("abstract_types.jl")
-
-# Generic benchmark framework
-include("generic_benchmark.jl")
-
-# Problem implementations
-include("factoring/factoring_problem.jl")
-include("factoring/factor_IP.jl")
-
-# Re-export main interfaces
-export AbstractBenchmarkProblem, AbstractProblemConfig, AbstractSolver
-export generate_instance, solve_instance, problem_id, default_configs, filename_pattern
+include("utils.jl")
+include("benchmark.jl")
+include("formatting.jl")
+include("comparison.jl")
+
+# CircuitIO
+include("circuitIO/circuitIO.jl")
+using .CircuitIO
+
+# Factoring problem
+include("factoring/types.jl")
+include("factoring/interface.jl")
+include("factoring/generators.jl")
+include("factoring/solver_ip.jl")
+include("factoring/solvers.jl")
+include("factoring/dataset.jl")
+include("factoring/solver_xsat.jl")
+
+export AbstractBenchmarkProblem, AbstractProblemConfig, AbstractInstance, AbstractSolver
+export solve_instance, verify_solution, read_instances
 export available_solvers, default_solver, solver_name
-export generate_datasets, benchmark_problem, run_full_benchmark, benchmark_backend
-export create_configs_from_tuples
-export list_available_solvers, run_solver_comparison
-export FactoringProblem, FactoringConfig, BooleanInferenceSolver, IPSolver
-export print_solver_comparison_summary, save_solver_comparison
+export benchmark_dataset, run_solver_comparison
+export list_available_solvers, print_solver_comparison_summary, compare_solver_results
+export FactoringProblem, FactoringConfig, FactoringInstance, generate_factoring_datasets
+export CircuitSATProblem, CircuitSATConfig, CircuitSATInstance
+export BooleanInferenceSolver, IPSolver, XSATSolver
 
-end # module
+end