feat(ci): add never-worked status for test regression classification

Author: pulk17

install/ci-vm/ci-linux/startup-script.sh

@@ -4,7 +4,7 @@ curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v3.2
 dpkg --install gcsfuse_3.2.0_amd64.deb
 rm gcsfuse_3.2.0_amd64.deb
 
-apt install gnupg ca-certificates
+apt install -y gnupg ca-certificates
 gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
 echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
 sudo apt update
@@ -14,7 +14,8 @@ mkdir repository
 cd repository
 
 # Use gcsfuse and import required files
-mkdir temp TestFiles TestResults vm_data reports
+# TempFiles is used by the tester (--tempfolder) and must exist
+mkdir temp TestFiles TestResults TempFiles vm_data reports
 
 gcs_bucket=$(curl http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google")
 
@@ -31,6 +32,9 @@ mount vm_data
 mount TestFiles
 mount TestResults
 
+# Give gcsfuse mounts time to become ready
+sleep 10
+
 cp temp/* ./
 
 chmod +x bootstrap

migrations/versions/d9f4e5a6b7c8_.py

@@ -0,0 +1,27 @@
+"""Add never_worked column to regression_test table
+
+Revision ID: d9f4e5a6b7c8
+Revises: c8f3a2b1d4e5
+Create Date: 2026-03-20 23:25:21.411651000000
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'd9f4e5a6b7c8'
+down_revision = 'c8f3a2b1d4e5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('regression_test', sa.Column('never_worked', sa.Boolean(), nullable=False, server_default='false'))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('regression_test', 'never_worked')
+    # ### end Alembic commands ###

mod_ci/controllers.py

@@ -45,7 +45,8 @@
 from mod_sample.models import Issue
 from mod_test.controllers import get_test_results
 from mod_test.models import (Fork, Test, TestPlatform, TestProgress,
-                             TestResult, TestResultFile, TestStatus, TestType)
+                             TestResult, TestResultFile, TestResultStatus,
+                             TestStatus, TestType)
 from utility import is_valid_signature, request_from_github
 
 # Timeout constants (in seconds)
@@ -2760,6 +2761,7 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo:
     extra_failed_tests = []
     common_failed_tests = []
     fixed_tests = []
+    never_worked_tests = []
     category_stats = []
 
     test_results = get_test_results(test)
@@ -2768,20 +2770,26 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo:
         category_name = category_results['category'].name
 
         category_test_pass_count = 0
-        for test in category_results['tests']:
-            if not test['error']:
+        for t in category_results['tests']:
+            if not t['error']:
                 category_test_pass_count += 1
-                if last_test_master and getattr(test['test'], platform_column) != last_test_master.id:
-                    fixed_tests.append(test['test'])
+                if last_test_master and getattr(t['test'], platform_column) != last_test_master.id:
+                    fixed_tests.append(t['test'])
             else:
-                if last_test_master and getattr(test['test'], platform_column) != last_test_master.id:
-                    common_failed_tests.append(test['test'])
+                # Separate out tests that have NEVER passed on any CCExtractor version
+                if t['status'] == TestResultStatus.never_worked:
+                    never_worked_tests.append(t['test'])
+                elif last_test_master and getattr(t['test'], platform_column) != last_test_master.id:
+                    common_failed_tests.append(t['test'])
                 else:
-                    extra_failed_tests.append(test['test'])
+                    extra_failed_tests.append(t['test'])
 
         category_stats.append(CategoryTestInfo(category_name, len(category_results['tests']), category_test_pass_count))
 
-    return PrCommentInfo(category_stats, extra_failed_tests, fixed_tests, common_failed_tests, last_test_master)
+    return PrCommentInfo(
+        category_stats, extra_failed_tests, fixed_tests, common_failed_tests,
+        last_test_master, never_worked_tests
+    )
 
 
 def comment_pr(test: Test) -> str:
@@ -2817,6 +2825,9 @@ def comment_pr(test: Test) -> str:
         log.debug(f"GitHub PR Comment ID {comment.id} Uploaded for Test_id: {test_id}")
     except Exception as e:
         log.error(f"GitHub PR Comment Failed for Test_id: {test_id} with Exception {e}")
+
+    # Determine PR status:
+    # SUCCESS if no regressions caused by PR (never_worked tests don't count)
     return Status.SUCCESS if len(comment_info.extra_failed_tests) == 0 else Status.FAILURE
 
 

mod_ci/models.py

@@ -179,3 +179,4 @@ class PrCommentInfo:
     fixed_tests: List[RegressionTest]
     common_failed_tests: List[RegressionTest]
     last_test_master: Test
+    never_worked_tests: List[RegressionTest]

mod_regression/controllers.py

@@ -160,6 +160,7 @@ def test_edit(regression_id):
         test.input_type = InputType.from_string(form.input_type.data)
         test.output_type = OutputType.from_string(form.output_type.data)
         test.description = form.description.data
+        test.never_worked = form.never_worked.data
 
         g.db.commit()
         g.log.info(f'regression test with id: {regression_id} updated!')
@@ -174,6 +175,7 @@ def test_edit(regression_id):
         form.input_type.data = test.input_type.value
         form.output_type.data = test.output_type.value
         form.description.data = test.description
+        form.never_worked.data = test.never_worked
 
     return {'form': form, 'regression_id': regression_id}
 
@@ -247,6 +249,7 @@ def test_add():
             input_type=InputType.from_string(form.input_type.data),
             output_type=OutputType.from_string(form.output_type.data),
             description=form.description.data,
+            never_worked=form.never_worked.data
         )
         g.db.add(new_test)
         category = Category.query.filter(Category.id == form.category_id.data).first()

mod_regression/forms.py

@@ -1,8 +1,8 @@
 """Maintain forms related to CRUD operations on regression tests."""
 
 from flask_wtf import FlaskForm
-from wtforms import (HiddenField, IntegerField, SelectField, StringField,
-                     SubmitField, TextAreaField)
+from wtforms import (BooleanField, HiddenField, IntegerField, SelectField,
+                     StringField, SubmitField, TextAreaField)
 from wtforms.validators import DataRequired, InputRequired, Length
 
 from mod_regression.models import InputType, OutputType
@@ -36,6 +36,7 @@ class CommonTestForm(FlaskForm):
     )
     category_id = SelectField("Category", coerce=int)
     expected_rc = IntegerField("Expected Runtime Code", [InputRequired(message="Expected Runtime Code can't be empty")])
+    never_worked = BooleanField("Never Worked", default=False)
 
 
 class AddTestForm(CommonTestForm):

mod_regression/models.py

@@ -97,9 +97,10 @@ class RegressionTest(Base):
     last_passed_on_windows = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL"))
     last_passed_on_linux = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL"))
     description = Column(String(length=1024))
+    never_worked = Column(Boolean(), default=False, nullable=False, server_default='false')
 
     def __init__(self, sample_id, command, input_type, output_type, category_id, expected_rc,
-                 active=True, description="") -> None:
+                 active=True, description="", never_worked=False) -> None:
         """
         Parametrized constructor for the RegressionTest model.
 
@@ -117,7 +118,10 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp
         :type expected_rc: int
         :param active: The value of the 'active' field of RegressionTest model
         :type active: bool
-
+        :param description: The value of the 'description' field of RegressionTest model
+        :type description: str
+        :param never_worked: Boolean flag whether the test has never worked for this sample
+        :type never_worked: bool
         """
         self.sample_id = sample_id
         self.command = command
@@ -127,6 +131,7 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp
         self.expected_rc = expected_rc
         self.active = active
         self.description = description
+        self.never_worked = never_worked
 
     def __repr__(self) -> str:
         """