Init cicd with commit 9bf277b

cicd/.github/workflows/giskard_action.yaml

@@ -0,0 +1,51 @@
+name: Deployment workflow
+
+on:
+  push:
+    paths:
+      - 'examples/github/train.py'
+      - 'examples/github/titanic_test_data.csv'
+      - 'examples/github/requirements.txt' # temporarily
+      - '.github/workflows/giskard_action.yaml' # temporarily
+
+jobs:
+  Deployment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+
+      - name: checkout repo content
+        uses: actions/checkout@v4 # checkout the repository content to github runner
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' # install the python version needed
+
+      - uses: syphar/restore-virtualenv@v1
+        id: cache-virtualenv
+        with:
+          requirement_files: examples/github/requirements.txt  # this is optional
+
+      - uses: syphar/restore-pip-download-cache@v1
+        if: steps.cache-virtualenv.outputs.cache-hit != 'true'
+
+        # the package installation will only be executed when the
+        # requirements-files have changed.
+      - run: pip install -r examples/github/requirements.txt
+        env:
+          EMAIL_ADDRESS: ${{ secrets.EMAIL_ADDRESS }}
+          EMAIL_PASSWORD: ${{ secrets.EMAIL_PASSWORD }}
+          EMAIL_RECIPIENT: ${{ secrets.EMAIL_RECIPIENT }}
+        if: steps.cache-virtualenv.outputs.cache-hit != 'true'
+
+      - name: training
+        run: |
+          python examples/github/train.py
+
+      - name: execute pipeline
+        run: |
+          python cli.py --loader github --model examples/github/artifacts/model --dataset examples/github/artifacts/dataset --output_format markdown

cicd/.models_and_datasets_to_be_skipped.csv

@@ -0,0 +1,4 @@
+model,dataset,status
+facebook/bart-large-mnli,multi_nli,error
+distilbert-base-uncased-finetuned-sst-2-english,sst2,done
+cardiffnlp/twitter-roberta-base-sentiment-latest,tweet_eval,done

cicd/automation/__init__.py

@@ -0,0 +1,3 @@
+from .post_discussion import create_discussion
+
+__all__ = ["create_discussion"]

cicd/automation/post_discussion.py

@@ -0,0 +1,5 @@
+import huggingface_hub as hf_hub
+def create_discussion(repo_id, model_name, hf_token, report):
+    # Create a discussion
+    discussion = hf_hub.create_discussion(repo_id, title=f"Report for {model_name}", token=hf_token, description=report, repo_type="space")
+    return discussion

cicd/cli.py

@@ -0,0 +1,69 @@
+import argparse
+
+from giskard_cicd.loaders import GithubLoader, HuggingFaceLoader
+from giskard_cicd.pipeline.runner import PipelineRunner
+
+from automation import create_discussion
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Giskard Scanner", description="Scans a model for vulnerabilities and produces a report."
+    )
+    parser.add_argument(
+        "--loader",
+        help="Which loader to use to set up the model. Currently only `github` and `huggingface` are supported.",
+        required=True,
+    )
+    parser.add_argument("--model", help="The model to scan.", required=True)
+    parser.add_argument("--dataset", help="The validation or test dataset that will be used.")
+    parser.add_argument(
+        "--dataset_split", help="The split of the dataset to use. If not provided, the best split will be selected."
+    )
+    parser.add_argument("--dataset_config", help="The name of the dataset config subset to use.")
+    parser.add_argument("--scan_config", help="Path to YAML file containing the configuration of the scan.")
+    parser.add_argument("--output", help="Optional name of the output file.")
+    parser.add_argument("--output_format", help="Format of the report (either HTML or markdown). Default is HTML.")
+    parser.add_argument("--output_portal", help="The output portal of the report (either huggingface or local directory). Default is local.")
+    parser.add_argument("--discussion_repo", help="The repo to push the report to.")
+    parser.add_argument("--hf_token", help="The token to push the report to the repo.")
+
+    args = parser.parse_args()
+
+    supported_loaders = {
+        "huggingface": HuggingFaceLoader(),
+        "github": GithubLoader(),
+    }
+
+    runner = PipelineRunner(loaders=supported_loaders)
+
+    runner_kwargs = {"loader_id": args.loader,
+                     "model": args.model,
+                     "dataset": args.dataset,
+                     "scan_config": args.scan_config}
+
+    if args.loader == "huggingface":
+        runner_kwargs.update({"dataset_split": args.dataset_split,
+                              "dataset_config": args.dataset_config})
+
+    report = runner.run(**runner_kwargs)
+
+    # In the future, write markdown report or directly push to discussion.
+    if args.output_format == "markdown":
+        rendered_report = report.to_markdown(template="github")
+    else:
+        rendered_report = report.to_html()
+    
+    if args.output_portal == "huggingface":
+        # Push to discussion
+        create_discussion(args.discussion_repo, args.model, args.hf_token, rendered_report)
+
+
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(rendered_report)
+    else:
+        # To stdout
+        # print(rendered_report)
+        model_name = args.model.split("/")[-1]
+        with open(f"{model_name}_report.html", "w") as f:
+            f.write(rendered_report)

cicd/examples/github/cicd_config.yaml

@@ -0,0 +1 @@
+artifact_path: "examples/github"

cicd/examples/github/readme.md

@@ -0,0 +1 @@
+# Github CI/CD

cicd/examples/github/requirements.txt

@@ -0,0 +1,6 @@
+giskard>=2.0.0b
+git+https://github.com/Giskard-AI/cicd.git@main
+json5==0.9.10
+jsonpatch==1.32
+jsonpointer==2.3
+jsonschema==3.2.0

cicd/examples/github/titanic_test_data.csv

@@ -0,0 +1,447 @@
+"PassengerId","Pclass","Name","Sex","Age","SibSp","Parch","Fare","Embarked","Survived"
+124,2,"Webber, Miss. Susan","female",32.5,0,0,13.0,"S","yes"
+715,2,"Greenberg, Mr. Samuel","male",52.0,0,0,13.0,"S","no"
+413,1,"Minahan, Miss. Daisy E","female",33.0,1,0,90.0,"Q","yes"
+82,3,"Sheerlinck, Mr. Jan Baptist","male",29.0,0,0,9.5,"S","yes"
+556,1,"Wright, Mr. George","male",62.0,0,0,26.55,"S","no"
+533,3,"Elias, Mr. Joseph Jr","male",17.0,1,1,7.2292,"C","no"
+850,1,"Goldenberg, Mrs. Samuel L (Edwiga Grabowska)","female","_GSK_NA_",1,0,89.1042,"C","yes"
+569,3,"Doharr, Mr. Tannous","male","_GSK_NA_",0,0,7.2292,"C","no"
+126,3,"Nicola-Yarred, Master. Elias","male",12.0,1,0,11.2417,"C","yes"
+544,2,"Beane, Mr. Edward","male",32.0,1,0,26.0,"S","yes"
+111,1,"Porter, Mr. Walter Chamberlain","male",47.0,0,0,52.0,"S","no"
+484,3,"Turkula, Mrs. (Hedwig)","female",63.0,0,0,9.5875,"S","yes"
+593,3,"Elsbury, Mr. William James","male",47.0,0,0,7.25,"S","no"
+422,3,"Charters, Mr. David","male",21.0,0,0,7.7333,"Q","no"
+847,3,"Sage, Mr. Douglas Bullen","male","_GSK_NA_",8,2,69.55,"S","no"
+328,2,"Ball, Mrs. (Ada E Hall)","female",36.0,0,0,13.0,"S","yes"
+828,2,"Mallet, Master. Andre","male",1.0,0,2,37.0042,"C","yes"
+883,3,"Dahlberg, Miss. Gerda Ulrika","female",22.0,0,0,10.5167,"S","no"
+437,3,"Ford, Miss. Doolina Margaret ""Daisy""","female",21.0,2,2,34.375,"S","no"
+88,3,"Slocovski, Mr. Selman Francis","male","_GSK_NA_",0,0,8.05,"S","no"
+705,3,"Hansen, Mr. Henrik Juul","male",26.0,1,0,7.8542,"S","no"
+391,1,"Carter, Mr. William Ernest","male",36.0,1,2,120.0,"S","yes"
+40,3,"Nicola-Yarred, Miss. Jamila","female",14.0,1,0,11.2417,"C","yes"
+672,1,"Davidson, Mr. Thornton","male",31.0,1,0,52.0,"S","no"
+620,2,"Gavey, Mr. Lawrence","male",26.0,0,0,10.5,"S","no"
+791,3,"Keane, Mr. Andrew ""Andy""","male","_GSK_NA_",0,0,7.75,"Q","no"
+63,1,"Harris, Mr. Henry Birkhardt","male",45.0,1,0,83.475,"S","no"
+800,3,"Van Impe, Mrs. Jean Baptiste (Rosalie Paula Govaert)","female",30.0,1,1,24.15,"S","no"
+317,2,"Kantor, Mrs. Sinai (Miriam Sternin)","female",24.0,1,0,26.0,"S","yes"
+617,3,"Danbom, Mr. Ernst Gilbert","male",34.0,1,1,14.4,"S","no"
+206,3,"Strom, Miss. Telma Matilda","female",2.0,0,1,10.4625,"S","no"
+274,1,"Natsch, Mr. Charles H","male",37.0,0,1,29.7,"C","no"
+567,3,"Stoytcheff, Mr. Ilia","male",19.0,0,0,7.8958,"S","no"
+632,3,"Lundahl, Mr. Johan Svensson","male",51.0,0,0,7.0542,"S","no"
+888,1,"Graham, Miss. Margaret Edith","female",19.0,0,0,30.0,"S","yes"
+480,3,"Hirvonen, Miss. Hildur E","female",2.0,0,1,12.2875,"S","yes"
+477,2,"Renouf, Mr. Peter Henry","male",34.0,1,0,21.0,"S","no"
+424,3,"Danbom, Mrs. Ernst Gilbert (Anna Sigrid Maria Brogren)","female",28.0,1,1,14.4,"S","no"
+741,1,"Hawksford, Mr. Walter James","male","_GSK_NA_",0,0,30.0,"S","yes"
+531,2,"Quick, Miss. Phyllis May","female",2.0,1,1,26.0,"S","yes"
+799,3,"Ibrahim Shawah, Mr. Yousseff","male",30.0,0,0,7.2292,"C","no"
+160,3,"Sage, Master. Thomas Henry","male","_GSK_NA_",8,2,69.55,"S","no"
+116,3,"Pekoniemi, Mr. Edvard","male",21.0,0,0,7.925,"S","no"
+290,3,"Connolly, Miss. Kate","female",22.0,0,0,7.75,"Q","yes"
+252,3,"Strom, Mrs. Wilhelm (Elna Matilda Persson)","female",29.0,1,1,10.4625,"S","no"
+306,1,"Allison, Master. Hudson Trevor","male",0.92,1,2,151.55,"S","yes"
+449,3,"Baclini, Miss. Marie Catherine","female",5.0,2,1,19.2583,"C","yes"
+483,3,"Rouse, Mr. Richard Henry","male",50.0,0,0,8.05,"S","no"
+587,2,"Jarvis, Mr. John Denzil","male",47.0,0,0,15.0,"S","no"
+25,3,"Palsson, Miss. Torborg Danira","female",8.0,3,1,21.075,"S","no"
+289,2,"Hosono, Mr. Masabumi","male",42.0,0,0,13.0,"S","yes"
+769,3,"Moran, Mr. Daniel J","male","_GSK_NA_",1,0,24.15,"Q","no"
+697,3,"Kelly, Mr. James","male",44.0,0,0,8.05,"S","no"
+172,3,"Rice, Master. Arthur","male",4.0,4,1,29.125,"Q","no"
+548,2,"Padro y Manent, Mr. Julian","male","_GSK_NA_",0,0,13.8625,"C","yes"
+586,1,"Taussig, Miss. Ruth","female",18.0,0,2,79.65,"S","yes"
+52,3,"Nosworthy, Mr. Richard Cater","male",21.0,0,0,7.8,"S","no"
+862,2,"Giles, Mr. Frederick Edward","male",21.0,1,0,11.5,"S","no"
+553,3,"O'Brien, Mr. Timothy","male","_GSK_NA_",0,0,7.8292,"Q","no"
+36,1,"Holverson, Mr. Alexander Oskar","male",42.0,1,0,52.0,"S","no"
+261,3,"Smith, Mr. Thomas","male","_GSK_NA_",0,0,7.75,"Q","no"
+366,3,"Adahl, Mr. Mauritz Nils Martin","male",30.0,0,0,7.25,"S","no"
+201,3,"Vande Walle, Mr. Nestor Cyriel","male",28.0,0,0,9.5,"S","no"
+761,3,"Garfirth, Mr. John","male","_GSK_NA_",0,0,14.5,"S","no"
+706,2,"Morley, Mr. Henry Samuel (""Mr Henry Marshall"")","male",39.0,0,0,26.0,"S","no"
+594,3,"Bourke, Miss. Mary","female","_GSK_NA_",0,2,7.75,"Q","no"
+53,1,"Harper, Mrs. Henry Sleeper (Myna Haxtun)","female",49.0,1,0,76.7292,"C","yes"
+546,1,"Nicholson, Mr. Arthur Ernest","male",64.0,0,0,26.0,"S","no"
+195,1,"Brown, Mrs. James Joseph (Margaret Tobin)","female",44.0,0,0,27.7208,"C","yes"
+530,2,"Hocking, Mr. Richard George","male",23.0,2,1,11.5,"S","no"
+702,1,"Silverthorne, Mr. Spencer Victor","male",35.0,0,0,26.2875,"S","yes"
+279,3,"Rice, Master. Eric","male",7.0,4,1,29.125,"Q","no"
+223,3,"Green, Mr. George Henry","male",51.0,0,0,8.05,"S","no"
+372,3,"Wiklund, Mr. Jakob Alfred","male",18.0,1,0,6.4958,"S","no"
+5,3,"Allen, Mr. William Henry","male",35.0,0,0,8.05,"S","no"
+519,2,"Angle, Mrs. William A (Florence ""Mary"" Agnes Hughes)","female",36.0,1,0,26.0,"S","yes"
+326,1,"Young, Miss. Marie Grice","female",36.0,0,0,135.6333,"C","yes"
+492,3,"Windelov, Mr. Einar","male",21.0,0,0,7.25,"S","no"
+344,2,"Sedgwick, Mr. Charles Frederick Waddington","male",25.0,0,0,13.0,"S","no"
+469,3,"Scanlan, Mr. James","male","_GSK_NA_",0,0,7.725,"Q","no"
+77,3,"Staneff, Mr. Ivan","male","_GSK_NA_",0,0,7.8958,"S","no"
+272,3,"Tornquist, Mr. William Henry","male",25.0,0,0,0.0,"S","yes"
+753,3,"Vande Velde, Mr. Johannes Joseph","male",33.0,0,0,9.5,"S","no"
+658,3,"Bourke, Mrs. John (Catherine)","female",32.0,1,1,15.5,"Q","no"
+388,2,"Buss, Miss. Kate","female",36.0,0,0,13.0,"S","yes"
+738,1,"Lesurer, Mr. Gustave J","male",35.0,0,0,512.3292,"C","yes"
+823,1,"Reuchlin, Jonkheer. John George","male",38.0,0,0,0.0,"S","no"
+814,3,"Andersson, Miss. Ebba Iris Alfrida","female",6.0,4,2,31.275,"S","no"
+596,3,"Van Impe, Mr. Jean Baptiste","male",36.0,1,1,24.15,"S","no"
+468,1,"Smart, Mr. John Montgomery","male",56.0,0,0,26.55,"S","no"
+95,3,"Coxon, Mr. Daniel","male",59.0,0,0,7.25,"S","no"
+148,3,"Ford, Miss. Robina Maggie ""Ruby""","female",9.0,2,2,34.375,"S","no"
+704,3,"Gallagher, Mr. Martin","male",25.0,0,0,7.7417,"Q","no"
+426,3,"Wiseman, Mr. Phillippe","male","_GSK_NA_",0,0,7.25,"S","no"
+730,3,"Ilmakangas, Miss. Pieta Sofia","female",25.0,1,0,7.925,"S","no"
+525,3,"Kassem, Mr. Fared","male","_GSK_NA_",0,0,7.2292,"C","no"
+727,2,"Renouf, Mrs. Peter Henry (Lillian Jefferys)","female",30.0,3,0,21.0,"S","yes"
+578,1,"Silvey, Mrs. William Baird (Alice Munger)","female",39.0,1,0,55.9,"S","yes"
+467,2,"Campbell, Mr. William","male","_GSK_NA_",0,0,0.0,"S","no"
+609,2,"Laroche, Mrs. Joseph (Juliette Marie Louise Lafargue)","female",22.0,1,2,41.5792,"C","yes"
+774,3,"Elias, Mr. Dibo","male","_GSK_NA_",0,0,7.225,"C","no"
+504,3,"Laitinen, Miss. Kristina Sofia","female",37.0,0,0,9.5875,"S","no"
+100,2,"Kantor, Mr. Sinai","male",34.0,1,0,26.0,"S","no"
+320,1,"Spedden, Mrs. Frederic Oakley (Margaretta Corning Stone)","female",40.0,1,1,134.5,"C","yes"
+98,1,"Greenfield, Mr. William Bertram","male",23.0,0,1,63.3583,"C","yes"
+880,1,"Potter, Mrs. Thomas Jr (Lily Alexenia Wilson)","female",56.0,0,1,83.1583,"C","yes"
+716,3,"Soholt, Mr. Peter Andreas Lauritz Andersen","male",19.0,0,0,7.65,"S","no"
+349,3,"Coutts, Master. William Loch ""William""","male",3.0,1,1,15.9,"S","yes"
+44,2,"Laroche, Miss. Simonne Marie Anne Andree","female",3.0,1,2,41.5792,"C","yes"
+631,1,"Barkworth, Mr. Algernon Henry Wilson","male",80.0,0,0,30.0,"S","yes"
+154,3,"van Billiard, Mr. Austin Blyler","male",40.5,0,2,14.5,"S","no"
+683,3,"Olsvigen, Mr. Thor Anderson","male",20.0,0,0,9.225,"S","no"
+92,3,"Andreasson, Mr. Paul Edvin","male",20.0,0,0,7.8542,"S","no"
+574,3,"Kelly, Miss. Mary","female","_GSK_NA_",0,0,7.75,"Q","yes"
+541,1,"Crosby, Miss. Harriet R","female",36.0,0,2,71.0,"S","yes"
+886,3,"Rice, Mrs. William (Margaret Norton)","female",39.0,0,5,29.125,"Q","no"
+215,3,"Kiernan, Mr. Philip","male","_GSK_NA_",1,0,7.75,"Q","no"
+381,1,"Bidois, Miss. Rosalie","female",42.0,0,0,227.525,"C","yes"
+776,3,"Myhrman, Mr. Pehr Fabian Oliver Malkolm","male",18.0,0,0,7.75,"S","no"
+430,3,"Pickard, Mr. Berk (Berk Trembisky)","male",32.0,0,0,8.05,"S","yes"
+222,2,"Bracken, Mr. James H","male",27.0,0,0,13.0,"S","no"
+820,3,"Skoog, Master. Karl Thorsten","male",10.0,3,2,27.9,"S","no"
+51,3,"Panula, Master. Juha Niilo","male",7.0,4,1,39.6875,"S","no"
+250,2,"Carter, Rev. Ernest Courtenay","male",54.0,1,0,26.0,"S","no"
+692,3,"Karun, Miss. Manca","female",4.0,0,1,13.4167,"C","yes"
+435,1,"Silvey, Mr. William Baird","male",50.0,1,0,55.9,"S","no"
+781,3,"Ayoub, Miss. Banoura","female",13.0,0,0,7.2292,"C","yes"
+491,3,"Hagland, Mr. Konrad Mathias Reiersen","male","_GSK_NA_",1,0,19.9667,"S","no"
+554,3,"Leeni, Mr. Fahim (""Philip Zenni"")","male",22.0,0,0,7.225,"C","yes"
+656,2,"Hickman, Mr. Leonard Mark","male",24.0,2,0,73.5,"S","no"
+380,3,"Gustafsson, Mr. Karl Gideon","male",19.0,0,0,7.775,"S","no"
+509,3,"Olsen, Mr. Henry Margido","male",28.0,0,0,22.525,"S","no"
+230,3,"Lefebre, Miss. Mathilde","female","_GSK_NA_",3,1,25.4667,"S","no"
+458,1,"Kenyon, Mrs. Frederick R (Marion)","female","_GSK_NA_",1,0,51.8625,"S","yes"
+733,2,"Knight, Mr. Robert J","male","_GSK_NA_",0,0,0.0,"S","no"
+740,3,"Nankoff, Mr. Minko","male","_GSK_NA_",0,0,7.8958,"S","no"
+420,3,"Van Impe, Miss. Catharina","female",10.0,0,2,24.15,"S","no"
+175,1,"Smith, Mr. James Clinch","male",56.0,0,0,30.6958,"C","no"
+767,1,"Brewe, Dr. Arthur Jackson","male","_GSK_NA_",0,0,39.6,"C","no"
+608,1,"Daniel, Mr. Robert Williams","male",27.0,0,0,30.5,"S","yes"
+75,3,"Bing, Mr. Lee","male",32.0,0,0,56.4958,"S","yes"
+242,3,"Murphy, Miss. Katherine ""Kate""","female","_GSK_NA_",1,0,15.5,"Q","yes"
+506,1,"Penasco y Castellana, Mr. Victor de Satode","male",18.0,1,0,108.9,"C","no"
+481,3,"Goodwin, Master. Harold Victor","male",9.0,5,2,46.9,"S","no"
+461,1,"Anderson, Mr. Harry","male",48.0,0,0,26.55,"S","yes"
+185,3,"Kink-Heilmann, Miss. Luise Gretchen","female",4.0,0,2,22.025,"S","yes"
+866,2,"Bystrom, Mrs. (Karolina)","female",42.0,0,0,13.0,"S","yes"
+165,3,"Panula, Master. Eino Viljami","male",1.0,4,1,39.6875,"S","no"
+406,2,"Gale, Mr. Shadrach","male",34.0,1,0,21.0,"S","no"
+248,2,"Hamalainen, Mrs. William (Anna)","female",24.0,0,2,14.5,"S","yes"
+211,3,"Ali, Mr. Ahmed","male",24.0,0,0,7.05,"S","no"
+337,1,"Pears, Mr. Thomas Clinton","male",29.0,1,0,66.6,"S","no"
+879,3,"Laleff, Mr. Kristo","male","_GSK_NA_",0,0,7.8958,"S","no"
+15,3,"Vestrom, Miss. Hulda Amanda Adolfina","female",14.0,0,0,7.8542,"S","no"
+56,1,"Woolner, Mr. Hugh","male","_GSK_NA_",0,0,35.5,"S","yes"
+302,3,"McCoy, Mr. Bernard","male","_GSK_NA_",2,0,23.25,"Q","yes"
+97,1,"Goldschmidt, Mr. George B","male",71.0,0,0,34.6542,"C","no"
+600,1,"Duff Gordon, Sir. Cosmo Edmund (""Mr Morgan"")","male",49.0,1,0,56.9292,"C","yes"
+876,3,"Najib, Miss. Adele Kiamie ""Jane""","female",15.0,0,0,7.225,"C","yes"
+731,1,"Allen, Miss. Elisabeth Walton","female",29.0,0,0,211.3375,"S","yes"
+744,3,"McNamee, Mr. Neal","male",24.0,1,0,16.1,"S","no"
+30,3,"Todoroff, Mr. Lalio","male","_GSK_NA_",0,0,7.8958,"S","no"
+673,2,"Mitchell, Mr. Henry Michael","male",70.0,0,0,10.5,"S","no"
+841,3,"Alhomaki, Mr. Ilmari Rudolf","male",20.0,0,0,7.925,"S","no"
+140,1,"Giglio, Mr. Victor","male",24.0,0,0,79.2,"C","no"
+62,1,"Icard, Miss. Amelie","female",38.0,0,0,80.0,"_GSK_NA_","yes"
+639,3,"Panula, Mrs. Juha (Maria Emilia Ojala)","female",41.0,0,5,39.6875,"S","no"
+693,3,"Lam, Mr. Ali","male","_GSK_NA_",0,0,56.4958,"S","yes"
+176,3,"Klasen, Mr. Klas Albin","male",18.0,1,1,7.8542,"S","no"
+417,2,"Drew, Mrs. James Vivian (Lulu Thorne Christian)","female",34.0,1,1,32.5,"S","yes"
+348,3,"Davison, Mrs. Thomas Henry (Mary E Finck)","female","_GSK_NA_",1,0,16.1,"S","yes"
+542,3,"Andersson, Miss. Ingeborg Constanzia","female",9.0,4,2,31.275,"S","no"
+433,2,"Louch, Mrs. Charles Alexander (Alice Adelaide Slow)","female",42.0,1,0,26.0,"S","yes"
+760,1,"Rothes, the Countess. of (Lucy Noel Martha Dyer-Edwards)","female",33.0,0,0,86.5,"S","yes"
+725,1,"Chambers, Mr. Norman Campbell","male",27.0,1,0,53.1,"S","yes"
+451,2,"West, Mr. Edwy Arthur","male",36.0,1,2,27.75,"S","no"
+91,3,"Christmann, Mr. Emil","male",29.0,0,0,8.05,"S","no"
+76,3,"Moen, Mr. Sigurd Hansen","male",25.0,0,0,7.65,"S","no"
+47,3,"Lennon, Mr. Denis","male","_GSK_NA_",1,0,15.5,"Q","no"
+65,1,"Stewart, Mr. Albert A","male","_GSK_NA_",0,0,27.7208,"C","no"
+258,1,"Cherry, Miss. Gladys","female",30.0,0,0,86.5,"S","yes"
+214,2,"Givard, Mr. Hans Kristensen","male",30.0,0,0,13.0,"S","no"
+245,3,"Attalah, Mr. Sleiman","male",30.0,0,0,7.225,"C","no"
+599,3,"Boulos, Mr. Hanna","male","_GSK_NA_",0,0,7.225,"C","no"
+400,2,"Trout, Mrs. William H (Jessie L)","female",28.0,0,0,12.65,"S","yes"
+772,3,"Jensen, Mr. Niels Peder","male",48.0,0,0,7.8542,"S","no"
+37,3,"Mamee, Mr. Hanna","male","_GSK_NA_",0,0,7.2292,"C","yes"
+114,3,"Jussila, Miss. Katriina","female",20.0,1,0,9.825,"S","no"
+853,3,"Boulos, Miss. Nourelain","female",9.0,1,1,15.2458,"C","no"
+676,3,"Edvardsson, Mr. Gustaf Hjalmar","male",18.0,0,0,7.775,"S","no"
+287,3,"de Mulder, Mr. Theodore","male",30.0,0,0,9.5,"S","yes"
+583,2,"Downton, Mr. William James","male",54.0,0,0,26.0,"S","no"
+71,2,"Jenkin, Mr. Stephen Curnow","male",32.0,0,0,10.5,"S","no"
+120,3,"Andersson, Miss. Ellis Anna Maria","female",2.0,4,2,31.275,"S","no"
+144,3,"Burke, Mr. Jeremiah","male",19.0,0,0,6.75,"Q","no"
+493,1,"Molson, Mr. Harry Markland","male",55.0,0,0,30.5,"S","no"
+870,3,"Johnson, Master. Harold Theodor","male",4.0,1,1,11.1333,"S","yes"
+869,3,"van Melkebeke, Mr. Philemon","male","_GSK_NA_",0,0,9.5,"S","no"
+13,3,"Saundercock, Mr. William Henry","male",20.0,0,0,8.05,"S","no"
+685,2,"Brown, Mr. Thomas William Solomon","male",60.0,1,1,39.0,"S","no"
+643,3,"Skoog, Miss. Margit Elizabeth","female",2.0,3,2,27.9,"S","no"
+87,3,"Ford, Mr. William Neal","male",16.0,1,3,34.375,"S","no"
+296,1,"Lewy, Mr. Ervin G","male","_GSK_NA_",0,0,27.7208,"C","no"
+694,3,"Saad, Mr. Khalil","male",25.0,0,0,7.225,"C","no"
+410,3,"Lefebre, Miss. Ida","female","_GSK_NA_",3,1,25.4667,"S","no"
+645,3,"Baclini, Miss. Eugenie","female",0.75,2,1,19.2583,"C","yes"
+803,1,"Carter, Master. William Thornton II","male",11.0,1,2,120.0,"S","yes"
+450,1,"Peuchen, Major. Arthur Godfrey","male",52.0,0,0,30.5,"S","yes"
+550,2,"Davies, Master. John Morgan Jr","male",8.0,1,1,36.75,"S","yes"
+352,1,"Williams-Lambert, Mr. Fletcher Fellows","male","_GSK_NA_",0,0,35.0,"S","no"
+580,3,"Jussila, Mr. Eiriik","male",32.0,0,0,7.925,"S","yes"
+319,1,"Wick, Miss. Mary Natalie","female",31.0,0,2,164.8667,"S","yes"
+831,3,"Yasbeck, Mrs. Antoni (Selini Alexander)","female",15.0,1,0,14.4542,"C","yes"
+777,3,"Tobin, Mr. Roger","male","_GSK_NA_",0,0,7.75,"Q","no"
+341,2,"Navratil, Master. Edmond Roger","male",2.0,1,1,26.0,"S","yes"
+871,3,"Balkic, Mr. Cerin","male",26.0,0,0,7.8958,"S","no"
+271,1,"Cairns, Mr. Alexander","male","_GSK_NA_",0,0,31.0,"S","no"
+755,2,"Herman, Mrs. Samuel (Jane Laver)","female",48.0,1,2,65.0,"S","yes"
+110,3,"Moran, Miss. Bertha","female","_GSK_NA_",1,0,24.15,"Q","yes"
+829,3,"McCormack, Mr. Thomas Joseph","male","_GSK_NA_",0,0,7.75,"Q","yes"
+448,1,"Seward, Mr. Frederic Kimber","male",34.0,0,0,26.55,"S","yes"
+33,3,"Glynn, Miss. Mary Agatha","female","_GSK_NA_",0,0,7.75,"Q","yes"
+465,3,"Maisner, Mr. Simon","male","_GSK_NA_",0,0,8.05,"S","no"
+427,2,"Clarke, Mrs. Charles V (Ada Maria Winfield)","female",28.0,1,0,26.0,"S","yes"
+204,3,"Youseff, Mr. Gerious","male",45.5,0,0,7.225,"C","no"
+431,1,"Bjornstrom-Steffansson, Mr. Mauritz Hakan","male",28.0,0,0,26.55,"S","yes"
+732,3,"Hassan, Mr. Houssein G N","male",11.0,0,0,18.7875,"C","no"
+787,3,"Sjoblom, Miss. Anna Sofia","female",18.0,0,0,7.4958,"S","yes"
+508,1,"Bradley, Mr. George (""George Arthur Brayton"")","male","_GSK_NA_",0,0,26.55,"S","yes"
+802,2,"Collyer, Mrs. Harvey (Charlotte Annie Tate)","female",31.0,1,1,26.25,"S","yes"
+310,1,"Francatelli, Miss. Laura Mabel","female",30.0,0,0,56.9292,"C","yes"
+107,3,"Salkjelsvik, Miss. Anna Kristine","female",21.0,0,0,7.65,"S","yes"
+299,1,"Saalfeld, Mr. Adolphe","male","_GSK_NA_",0,0,30.5,"S","yes"
+459,2,"Toomey, Miss. Ellen","female",50.0,0,0,10.5,"S","yes"
+641,3,"Jensen, Mr. Hans Peder","male",20.0,0,0,7.8542,"S","no"
+668,3,"Rommetvedt, Mr. Knud Paust","male","_GSK_NA_",0,0,7.775,"S","no"
+523,3,"Lahoud, Mr. Sarkis","male","_GSK_NA_",0,0,7.225,"C","no"
+710,3,"Moubarek, Master. Halim Gonios (""William George"")","male","_GSK_NA_",1,1,15.2458,"C","yes"
+249,1,"Beckwith, Mr. Richard Leonard","male",37.0,1,1,52.5542,"S","yes"
+677,3,"Sawyer, Mr. Frederick Charles","male",24.5,0,0,8.05,"S","no"
+595,2,"Chapman, Mr. John Henry","male",37.0,1,0,26.0,"S","no"
+667,2,"Butler, Mr. Reginald Fenton","male",25.0,0,0,13.0,"S","no"
+537,1,"Butt, Major. Archibald Willingham","male",45.0,0,0,26.55,"S","no"
+666,2,"Hickman, Mr. Lewis","male",32.0,2,0,73.5,"S","no"
+581,2,"Christy, Miss. Julie Rachel","female",25.0,1,1,30.0,"S","yes"
+630,3,"O'Connell, Mr. Patrick D","male","_GSK_NA_",0,0,7.7333,"Q","no"
+648,1,"Simonius-Blumer, Col. Oberst Alfons","male",56.0,0,0,35.5,"C","yes"
+878,3,"Petroff, Mr. Nedelio","male",19.0,0,0,7.8958,"S","no"
+269,1,"Graham, Mrs. William Thompson (Edith Junkins)","female",58.0,0,1,153.4625,"S","yes"
+234,3,"Asplund, Miss. Lillian Gertrud","female",5.0,4,2,31.3875,"S","yes"
+644,3,"Foo, Mr. Choong","male","_GSK_NA_",0,0,56.4958,"S","yes"
+118,2,"Turpin, Mr. William John Robert","male",29.0,1,0,21.0,"S","no"
+333,1,"Graham, Mr. George Edward","male",38.0,0,1,153.4625,"S","no"
+454,1,"Goldenberg, Mr. Samuel L","male",49.0,1,0,89.1042,"C","yes"
+139,3,"Osen, Mr. Olaf Elon","male",16.0,0,0,9.2167,"S","no"
+606,3,"Lindell, Mr. Edvard Bengtsson","male",36.0,1,0,15.55,"S","no"
+535,3,"Cacic, Miss. Marija","female",30.0,0,0,8.6625,"S","no"
+221,3,"Sunderland, Mr. Victor Francis","male",16.0,0,0,8.05,"S","yes"
+444,2,"Reynaldo, Ms. Encarnacion","female",28.0,0,0,13.0,"S","yes"
+330,1,"Hippach, Miss. Jean Gertrude","female",16.0,0,1,57.9792,"C","yes"
+805,3,"Hedman, Mr. Oskar Arvid","male",27.0,0,0,6.975,"S","yes"
+55,1,"Ostby, Mr. Engelhart Cornelius","male",65.0,0,1,61.9792,"C","no"
+528,1,"Farthing, Mr. John","male","_GSK_NA_",0,0,221.7792,"S","no"
+359,3,"McGovern, Miss. Mary","female","_GSK_NA_",0,0,7.8792,"Q","yes"
+354,3,"Arnold-Franchi, Mr. Josef","male",25.0,1,0,17.8,"S","no"
+678,3,"Turja, Miss. Anna Sofia","female",18.0,0,0,9.8417,"S","yes"
+273,2,"Mellinger, Mrs. (Elizabeth Anne Maidment)","female",41.0,0,1,19.5,"S","yes"
+429,3,"Flynn, Mr. James","male","_GSK_NA_",0,0,7.75,"Q","no"
+536,2,"Hart, Miss. Eva Miriam","female",7.0,0,2,26.25,"S","yes"
+838,3,"Sirota, Mr. Maurice","male","_GSK_NA_",0,0,8.05,"S","no"
+179,2,"Hale, Mr. Reginald","male",30.0,0,0,13.0,"S","no"
+339,3,"Dahl, Mr. Karl Edwart","male",45.0,0,0,8.05,"S","yes"
+724,2,"Hodges, Mr. Henry Price","male",50.0,0,0,13.0,"S","no"
+524,1,"Hippach, Mrs. Louis Albert (Ida Sophia Fischer)","female",44.0,0,1,57.9792,"C","yes"
+734,2,"Berriman, Mr. William John","male",23.0,0,0,13.0,"S","no"
+164,3,"Calic, Mr. Jovo","male",17.0,0,0,8.6625,"S","no"
+304,2,"Keane, Miss. Nora A","female","_GSK_NA_",0,0,12.35,"Q","yes"
+356,3,"Vanden Steen, Mr. Leo Peter","male",28.0,0,0,9.5,"S","no"
+436,1,"Carter, Miss. Lucile Polk","female",14.0,1,2,120.0,"S","yes"
+622,1,"Kimball, Mr. Edwin Nelson Jr","male",42.0,1,0,52.5542,"S","yes"
+551,1,"Thayer, Mr. John Borland Jr","male",17.0,0,2,110.8833,"C","yes"
+109,3,"Rekic, Mr. Tido","male",38.0,0,0,7.8958,"S","no"
+265,3,"Henry, Miss. Delia","female","_GSK_NA_",0,0,7.75,"Q","no"
+628,1,"Longley, Miss. Gretchen Fiske","female",21.0,0,0,77.9583,"S","yes"
+394,1,"Newell, Miss. Marjorie","female",23.0,1,0,113.275,"C","yes"
+748,2,"Sinkkonen, Miss. Anna","female",30.0,0,0,13.0,"S","yes"
+698,3,"Mullens, Miss. Katherine ""Katie""","female","_GSK_NA_",0,0,7.7333,"Q","yes"
+66,3,"Moubarek, Master. Gerios","male","_GSK_NA_",1,1,15.2458,"C","yes"
+681,3,"Peters, Miss. Katie","female","_GSK_NA_",0,0,8.1375,"Q","no"
+663,1,"Colley, Mr. Edward Pomeroy","male",47.0,0,0,25.5875,"S","no"
+158,3,"Corn, Mr. Harry","male",30.0,0,0,8.05,"S","no"
+298,1,"Allison, Miss. Helen Loraine","female",2.0,1,2,151.55,"S","no"
+674,2,"Wilhelms, Mr. Charles","male",31.0,0,0,13.0,"S","yes"
+808,3,"Pettersson, Miss. Ellen Natalia","female",18.0,0,0,7.775,"S","no"
+545,1,"Douglas, Mr. Walter Donald","male",50.0,1,0,106.425,"C","no"
+338,1,"Burns, Miss. Elizabeth Margaret","female",41.0,0,0,134.5,"C","yes"
+833,3,"Saad, Mr. Amin","male","_GSK_NA_",0,0,7.2292,"C","no"
+94,3,"Dean, Mr. Bertram Frank","male",26.0,1,2,20.575,"S","no"
+133,3,"Robins, Mrs. Alexander A (Grace Charity Laury)","female",47.0,1,0,14.5,"S","no"
+383,3,"Tikkanen, Mr. Juho","male",32.0,0,0,7.925,"S","no"
+720,3,"Johnson, Mr. Malkolm Joackim","male",33.0,0,0,7.775,"S","no"
+739,3,"Ivanoff, Mr. Kanio","male","_GSK_NA_",0,0,7.8958,"S","no"
+343,2,"Collander, Mr. Erik Gustaf","male",28.0,0,0,13.0,"S","no"
+647,3,"Cor, Mr. Liudevit","male",19.0,0,0,7.8958,"S","no"
+286,3,"Stankovic, Mr. Ivan","male",33.0,0,0,8.6625,"C","no"
+743,1,"Ryerson, Miss. Susan Parker ""Suzette""","female",21.0,2,2,262.375,"C","yes"
+371,1,"Harder, Mr. George Achilles","male",25.0,1,0,55.4417,"C","yes"
+457,1,"Millet, Mr. Francis Davis","male",65.0,0,0,26.55,"S","no"
+882,3,"Markun, Mr. Johann","male",33.0,0,0,7.8958,"S","no"
+884,2,"Banfield, Mr. Frederick James","male",28.0,0,0,10.5,"S","no"
+560,3,"de Messemaeker, Mrs. Guillaume Joseph (Emma)","female",36.0,1,0,17.4,"S","yes"
+168,3,"Skoog, Mrs. William (Anna Bernhardina Karlsson)","female",45.0,1,4,27.9,"S","no"
+636,2,"Davis, Miss. Mary","female",28.0,0,0,13.0,"S","yes"
+885,3,"Sutehall, Mr. Henry Jr","male",25.0,0,0,7.05,"S","no"
+131,3,"Drazenoic, Mr. Jozef","male",33.0,0,0,7.8958,"C","no"
+505,1,"Maioni, Miss. Roberta","female",16.0,0,0,86.5,"S","yes"
+332,1,"Partner, Mr. Austen","male",45.5,0,0,28.5,"S","no"
+132,3,"Coelho, Mr. Domingos Fernandeo","male",20.0,0,0,7.05,"S","no"
+500,3,"Svensson, Mr. Olof","male",24.0,0,0,7.7958,"S","no"
+135,2,"Sobey, Mr. Samuel James Hayden","male",25.0,0,0,13.0,"S","no"
+192,2,"Carbines, Mr. William","male",19.0,0,0,13.0,"S","no"
+61,3,"Sirayanian, Mr. Orsen","male",22.0,0,0,7.2292,"C","no"
+819,3,"Holm, Mr. John Fredrik Alexander","male",43.0,0,0,6.45,"S","no"
+428,2,"Phillips, Miss. Kate Florence (""Mrs Kate Louise Phillips Marshall"")","female",19.0,0,0,26.0,"S","yes"
+161,3,"Cribb, Mr. John Hatfield","male",44.0,0,1,16.1,"S","no"
+117,3,"Connors, Mr. Patrick","male",70.5,0,0,7.75,"Q","no"
+839,3,"Chip, Mr. Chang","male",32.0,0,0,56.4958,"S","yes"
+861,3,"Hansen, Mr. Claus Peter","male",41.0,2,0,14.1083,"S","no"
+688,3,"Dakic, Mr. Branko","male",19.0,0,0,10.1708,"S","no"
+283,3,"de Pelsmaeker, Mr. Alfons","male",16.0,0,0,9.5,"S","no"
+402,3,"Adams, Mr. John","male",26.0,0,0,8.05,"S","no"
+843,1,"Serepeca, Miss. Augusta","female",30.0,0,0,31.0,"C","yes"
+48,3,"O'Driscoll, Miss. Bridget","female","_GSK_NA_",0,0,7.75,"Q","yes"
+770,3,"Gronnestad, Mr. Daniel Danielsen","male",32.0,0,0,8.3625,"S","no"
+405,3,"Oreskovic, Miss. Marija","female",20.0,0,0,8.6625,"S","no"
+874,3,"Vander Cruyssen, Mr. Victor","male",47.0,0,0,9.0,"S","no"
+196,1,"Lurette, Miss. Elise","female",58.0,0,0,146.5208,"C","yes"
+167,1,"Chibnall, Mrs. (Edith Martha Bowerman)","female","_GSK_NA_",0,1,55.0,"S","yes"
+517,2,"Lemore, Mrs. (Amelia Milley)","female",34.0,0,0,10.5,"S","yes"
+526,3,"Farrell, Mr. James","male",40.5,0,0,7.75,"Q","no"
+473,2,"West, Mrs. Edwy Arthur (Ada Mary Worth)","female",33.0,1,2,27.75,"S","yes"
+113,3,"Barton, Mr. David John","male",22.0,0,0,8.05,"S","no"
+701,1,"Astor, Mrs. John Jacob (Madeleine Talmadge Force)","female",18.0,1,0,227.525,"C","yes"
+369,3,"Jermyn, Miss. Annie","female","_GSK_NA_",0,0,7.75,"Q","yes"
+779,3,"Kilgannon, Mr. Thomas J","male","_GSK_NA_",0,0,7.7375,"Q","no"
+475,3,"Strandberg, Miss. Ida Sofia","female",22.0,0,0,9.8375,"S","no"
+184,2,"Becker, Master. Richard F","male",1.0,2,1,39.0,"S","yes"
+707,2,"Kelly, Mrs. Florence ""Fannie""","female",45.0,0,0,13.5,"S","yes"
+136,2,"Richard, Mr. Emile","male",23.0,0,0,15.0458,"C","no"
+865,2,"Gill, Mr. John William","male",24.0,0,0,13.0,"S","no"
+364,3,"Asim, Mr. Adola","male",35.0,0,0,7.05,"S","no"
+149,2,"Navratil, Mr. Michel (""Louis M Hoffman"")","male",36.5,0,2,26.0,"S","no"
+789,3,"Dean, Master. Bertram Vere","male",1.0,1,2,20.575,"S","yes"
+745,3,"Stranden, Mr. Juho","male",31.0,0,0,7.925,"S","yes"
+293,2,"Levy, Mr. Rene Jacques","male",36.0,0,0,12.875,"C","no"
+726,3,"Oreskovic, Mr. Luka","male",20.0,0,0,8.6625,"S","no"
+679,3,"Goodwin, Mrs. Frederick (Augusta Tyler)","female",43.0,1,6,46.9,"S","no"
+476,1,"Clifford, Mr. George Quincy","male","_GSK_NA_",0,0,52.0,"S","no"
+157,3,"Gilnagh, Miss. Katherine ""Katie""","female",16.0,0,0,7.7333,"Q","yes"
+875,2,"Abelson, Mrs. Samuel (Hannah Wizosky)","female",28.0,1,0,24.0,"C","yes"
+193,3,"Andersen-Jensen, Miss. Carla Christine Nielsine","female",19.0,1,0,7.8542,"S","yes"
+357,1,"Bowerman, Miss. Elsie Edith","female",22.0,0,1,55.0,"S","yes"
+610,1,"Shutes, Miss. Elizabeth W","female",40.0,0,0,153.4625,"S","yes"
+568,3,"Palsson, Mrs. Nils (Alma Cornelia Berglund)","female",29.0,0,4,21.075,"S","no"
+634,1,"Parr, Mr. William Henry Marsh","male","_GSK_NA_",0,0,0.0,"S","no"
+18,2,"Williams, Mr. Charles Eugene","male","_GSK_NA_",0,0,13.0,"S","yes"
+751,2,"Wells, Miss. Joan","female",4.0,1,1,23.0,"S","yes"
+128,3,"Madsen, Mr. Fridtjof Arne","male",24.0,0,0,7.1417,"S","yes"
+38,3,"Cann, Mr. Ernest Charles","male",21.0,0,0,8.05,"S","no"
+564,3,"Simmons, Mr. John","male","_GSK_NA_",0,0,8.05,"S","no"
+224,3,"Nenkoff, Mr. Christo","male","_GSK_NA_",0,0,7.8958,"S","no"
+266,2,"Reeves, Mr. David","male",36.0,0,0,10.5,"S","no"
+397,3,"Olsson, Miss. Elina","female",31.0,0,0,7.8542,"S","no"
+754,3,"Jonkoff, Mr. Lalio","male",23.0,0,0,7.8958,"S","no"
+412,3,"Hart, Mr. Henry","male","_GSK_NA_",0,0,6.8583,"Q","no"
+890,1,"Behr, Mr. Karl Howell","male",26.0,0,0,30.0,"C","yes"
+709,1,"Cleaver, Miss. Alice","female",22.0,0,0,151.55,"S","yes"
+818,2,"Mallet, Mr. Albert","male",31.0,1,1,37.0042,"C","no"
+336,3,"Denkoff, Mr. Mitto","male","_GSK_NA_",0,0,7.8958,"S","no"
+809,2,"Meyer, Mr. August","male",39.0,0,0,13.0,"S","no"
+373,3,"Beavan, Mr. William Thomas","male",19.0,0,0,8.05,"S","no"
+311,1,"Hays, Miss. Margaret Bechstein","female",24.0,0,0,83.1583,"C","yes"
+181,3,"Sage, Miss. Constance Gladys","female","_GSK_NA_",8,2,69.55,"S","no"
+392,3,"Jansson, Mr. Carl Olof","male",21.0,0,0,7.7958,"S","yes"
+496,3,"Yousseff, Mr. Gerious","male","_GSK_NA_",0,0,14.4583,"C","no"
+81,3,"Waelens, Mr. Achille","male",22.0,0,0,9.0,"S","no"
+125,1,"White, Mr. Percival Wayland","male",54.0,0,1,77.2875,"S","no"
+301,3,"Kelly, Miss. Anna Katherine ""Annie Kate""","female","_GSK_NA_",0,0,7.75,"Q","yes"
+816,1,"Fry, Mr. Richard","male","_GSK_NA_",0,0,0.0,"S","no"
+794,1,"Hoyt, Mr. William Fisher","male","_GSK_NA_",0,0,30.6958,"C","no"
+867,2,"Duran y More, Miss. Asuncion","female",27.0,1,0,13.8583,"C","yes"
+759,3,"Theobald, Mr. Thomas Leonard","male",34.0,0,0,8.05,"S","no"
+793,3,"Sage, Miss. Stella Anna","female","_GSK_NA_",8,2,69.55,"S","no"
+764,1,"Carter, Mrs. William Ernest (Lucile Polk)","female",36.0,1,2,120.0,"S","yes"
+687,3,"Panula, Mr. Jaako Arnold","male",14.0,4,1,39.6875,"S","no"
+246,1,"Minahan, Dr. William Edward","male",44.0,2,0,90.0,"Q","no"
+309,2,"Abelson, Mr. Samuel","male",30.0,1,0,24.0,"C","no"
+708,1,"Calderhead, Mr. Edward Pennington","male",42.0,0,0,26.2875,"S","yes"
+848,3,"Markoff, Mr. Marin","male",35.0,0,0,7.8958,"C","no"
+825,3,"Panula, Master. Urho Abraham","male",2.0,4,1,39.6875,"S","no"
+690,1,"Madill, Miss. Georgette Alexandra","female",15.0,0,1,211.3375,"S","yes"
+385,3,"Plotcharsky, Mr. Vasil","male","_GSK_NA_",0,0,7.8958,"S","no"
+758,2,"Bailey, Mr. Percy Andrew","male",18.0,0,0,11.5,"S","no"
+233,2,"Sjostedt, Mr. Ernst Adolf","male",59.0,0,0,13.5,"S","no"
+651,3,"Mitkoff, Mr. Mito","male","_GSK_NA_",0,0,7.8958,"S","no"
+616,2,"Herman, Miss. Alice","female",24.0,1,2,65.0,"S","yes"
+19,3,"Vander Planke, Mrs. Julius (Emelia Maria Vandemoortele)","female",31.0,1,0,18.0,"S","no"
+183,3,"Asplund, Master. Clarence Gustaf Hugo","male",9.0,4,2,31.3875,"S","no"
+597,2,"Leitch, Miss. Jessie Wills","female","_GSK_NA_",0,0,33.0,"S","yes"
+463,1,"Gee, Mr. Arthur H","male",47.0,0,0,38.5,"S","no"
+67,2,"Nye, Mrs. (Elizabeth Ramell)","female",29.0,0,0,10.5,"S","yes"
+788,3,"Rice, Master. George Hugh","male",8.0,4,1,29.125,"Q","no"
+518,3,"Ryan, Mr. Patrick","male","_GSK_NA_",0,0,24.15,"Q","no"
+104,3,"Johansson, Mr. Gustaf Joel","male",33.0,0,0,8.6542,"S","no"
+729,2,"Bryhl, Mr. Kurt Arnold Gottfrid","male",25.0,1,0,26.0,"S","no"
+8,3,"Palsson, Master. Gosta Leonard","male",2.0,3,1,21.075,"S","no"
+812,3,"Lester, Mr. James","male",39.0,0,0,24.15,"S","no"
+502,3,"Canavan, Miss. Mary","female",21.0,0,0,7.75,"Q","no"
+614,3,"Horgan, Mr. John","male","_GSK_NA_",0,0,7.75,"Q","no"
+34,2,"Wheadon, Mr. Edward H","male",66.0,0,0,10.5,"S","no"
+294,3,"Haas, Miss. Aloisia","female",24.0,0,0,8.85,"S","no"
+323,2,"Slayter, Miss. Hilda Mary","female",30.0,0,0,12.35,"Q","yes"
+652,2,"Doling, Miss. Elsie","female",18.0,0,1,23.0,"S","yes"
+827,3,"Lam, Mr. Len","male","_GSK_NA_",0,0,56.4958,"S","no"
+331,3,"McCoy, Miss. Agnes","female","_GSK_NA_",2,0,23.25,"Q","yes"
+439,1,"Fortune, Mr. Mark","male",64.0,1,4,263.0,"S","no"
+798,3,"Osman, Mrs. Mara","female",31.0,0,0,8.6833,"S","yes"
+623,3,"Nakid, Mr. Sahid","male",20.0,1,1,15.7417,"C","yes"
+276,1,"Andrews, Miss. Kornelia Theodosia","female",63.0,1,0,77.9583,"S","yes"
+78,3,"Moutal, Mr. Rahamin Haim","male","_GSK_NA_",0,0,8.05,"S","no"
+742,1,"Cavendish, Mr. Tyrell William","male",36.0,1,0,78.85,"S","no"
+370,1,"Aubart, Mme. Leontine Pauline","female",24.0,0,0,69.3,"C","yes"
+425,3,"Rosblom, Mr. Viktor Richard","male",18.0,1,1,20.2125,"S","no"
+189,3,"Bourke, Mr. John","male",40.0,1,1,15.5,"Q","no"
+143,3,"Hakkarainen, Mrs. Pekka Pietari (Elin Matilda Dolck)","female",24.0,1,0,15.85,"S","yes"
+627,2,"Kirkland, Rev. Charles Leonard","male",57.0,0,0,12.35,"Q","no"
+703,3,"Barbara, Miss. Saiide","female",18.0,0,1,14.4542,"C","no"
+638,2,"Collyer, Mr. Harvey","male",31.0,1,1,26.25,"S","no"
+549,3,"Goldsmith, Mr. Frank John","male",33.0,1,1,20.525,"S","no"
+43,3,"Kraeff, Mr. Theodor","male","_GSK_NA_",0,0,7.8958,"C","no"
+68,3,"Crease, Mr. Ernest James","male",19.0,0,0,8.1583,"S","no"
+756,2,"Hamalainen, Master. Viljo","male",0.67,1,1,14.5,"S","yes"
+443,3,"Petterson, Mr. Johan Emil","male",25.0,1,0,7.775,"S","no"
+472,3,"Cacic, Mr. Luka","male",38.0,0,0,8.6625,"S","no"
+696,2,"Chapman, Mr. Charles Henry","male",52.0,0,0,13.5,"S","no"
+665,3,"Lindqvist, Mr. Eino William","male",20.0,1,0,7.925,"S","yes"

cicd/examples/github/train.py

@@ -0,0 +1,38 @@
+import giskard
+import yaml
+
+path_to_config = __file__.split("train.py")[0]+"cicd_config.yaml"
+with open(path_to_config) as yaml_f:
+    cicd_config = yaml.load(yaml_f, Loader=yaml.Loader)
+
+# Replace this with your own data & model creation.
+df = giskard.demo.titanic_df()
+data_preprocessor, clf = giskard.demo.titanic_pipeline()
+
+# Wrap your Pandas DataFrame with Giskard.Dataset (test set, a golden dataset, etc.). Check the dedicated doc page: https://docs.giskard.ai/en/latest/guides/wrap_dataset/index.html
+giskard_dataset = giskard.Dataset(
+    df=df,  # A pandas.DataFrame that contains the raw data (before all the pre-processing steps) and the actual ground truth variable (target).
+    target="Survived",  # Ground truth variable
+    name="Titanic dataset", # Optional
+    cat_columns=['Pclass', 'Sex', "SibSp", "Parch", "Embarked"]  # Optional, but is a MUST if available. Inferred automatically if not.
+)
+
+# Wrap your model with Giskard.Model. Check the dedicated doc page: https://docs.giskard.ai/en/latest/guides/wrap_model/index.html
+# you can use any tabular, text or LLM models (PyTorch, HuggingFace, LangChain, etc.),
+# for classification, regression & text generation.
+def prediction_function(df):
+    # The pre-processor can be a pipeline of one-hot encoding, imputer, scaler, etc.
+    preprocessed_df = data_preprocessor(df)
+    return clf.predict_proba(preprocessed_df)
+
+giskard_model = giskard.Model(
+    model=prediction_function,  # A prediction function that encapsulates all the data pre-processing steps and that could be executed with the dataset used by the scan.
+    model_type="classification",  # Either regression, classification or text_generation.
+    name="Titanic model",  # Optional
+    classification_labels=clf.classes_,  # Their order MUST be identical to the prediction_function's output order
+    feature_names=['PassengerId', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked'],  # Default: all columns of your dataset
+    # classification_threshold=0.5,  # Default: 0.5
+)
+
+from giskard_cicd.utils import dump_model_and_dataset_for_cicd
+dump_model_and_dataset_for_cicd(cicd_config["artifact_path"], giskard_model, giskard_dataset)

cicd/giskard_cicd/__init__.py

@@ -0,0 +1,3 @@
+from .utils import dump_model_and_dataset_for_cicd
+
+__all__ = ["dump_model_and_dataset_for_cicd"]

cicd/giskard_cicd/loaders/__init__.py

@@ -0,0 +1,5 @@
+from giskard_cicd.loaders.github_loader import GithubLoader
+from giskard_cicd.loaders.huggingface_loader import HuggingFaceLoader
+from giskard_cicd.loaders.base_loader import BaseLoader
+
+__all__ = ["GithubLoader", "HuggingFaceLoader", "BaseLoader"]

cicd/giskard_cicd/loaders/base_loader.py

@@ -0,0 +1,33 @@
+"""Load models and datasets from Github."""
+
+import logging
+from abc import ABC, abstractmethod
+
+from giskard.models.base import BaseModel
+from giskard.core.model_validation import validate_model
+from giskard import Dataset
+
+logger = logging.getLogger(__name__)
+
+
+class LoaderError(RuntimeError):
+    """Could not load the model and/or dataset."""
+
+
+class DatasetError(LoaderError):
+    """Problems related to the dataset."""
+
+
+class ModelError(LoaderError):
+    """Problems related to the model."""
+
+
+class BaseLoader(ABC):
+
+    @abstractmethod
+    def load_giskard_model_dataset(self) -> (BaseModel, Dataset):
+        ...
+
+    def validate(self):
+        gsk_model, gsk_dataset = self.load_giskard_model_dataset()
+        validate_model(gsk_model, gsk_dataset)

cicd/giskard_cicd/loaders/github_loader.py

@@ -0,0 +1,33 @@
+import yaml
+from giskard import Dataset, Model
+from giskard.models.base import BaseModel
+from giskard.ml_worker.utils.file_utils import get_file_name
+
+from .base_loader import BaseLoader
+from pathlib import Path
+from giskard.core.core import DatasetMeta
+
+
+class GithubLoader(BaseLoader):
+    #TODO: change the way dataset is loaded, factor out some of the logic contained in Dataset.download()
+    def load_giskard_model_dataset(self, model, dataset) -> (BaseModel, Dataset):
+        with open(Path(dataset) / "giskard-dataset-meta.yaml") as f:
+            saved_meta = yaml.load(f, Loader=yaml.Loader)
+            meta = DatasetMeta(
+                name=saved_meta["name"],
+                target=saved_meta["target"],
+                column_types=saved_meta["column_types"],
+                column_dtypes=saved_meta["column_dtypes"],
+                number_of_rows=saved_meta["number_of_rows"],
+                category_features=saved_meta["category_features"],
+            )
+
+        df = Dataset.load(Path(dataset) / get_file_name("data", "csv.zst", False))
+        df = Dataset.cast_column_to_dtypes(df, meta.column_dtypes)
+
+        return Model.load(model), Dataset(
+            df=df,
+            name=meta.name,
+            target=meta.target,
+            column_types=meta.column_types,
+        )

cicd/giskard_cicd/loaders/huggingface_loader.py

@@ -0,0 +1,254 @@
+"""Load models and datasets from the HuggingFace hub."""
+
+import logging
+import time
+
+import datasets
+import giskard as gsk
+import huggingface_hub
+import torch
+from giskard import Dataset
+from giskard.models.base import BaseModel
+from giskard.models.huggingface import HuggingFaceModel
+from transformers.pipelines import TextClassificationPipeline
+import pandas as pd
+from .base_loader import BaseLoader, DatasetError
+
+logger = logging.getLogger(__name__)
+
+
+class HuggingFaceLoader(BaseLoader):
+
+    def __init__(self, device=None):
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+
+    def _find_dataset_id_from_model(self, model_id):
+        """Find the dataset ID from the model metadata."""
+        model_card = huggingface_hub.model_info(model_id).cardData
+
+        if "datasets" not in model_card:
+            msg = f"Could not find dataset for model `{model_id}`."
+            raise DatasetError(msg)
+
+        # Take the first one
+        dataset_id = model_card["datasets"][0]
+        return dataset_id
+
+    def load_giskard_model_dataset(self, model, dataset=None, dataset_config=None, dataset_split=None):
+        # If no dataset was provided, we try to get it from the model metadata.
+        if dataset is None:
+            logger.debug("No dataset provided. Trying to get it from the model metadata.")
+            dataset = self._find_dataset_id_from_model(model)
+            logger.debug(f"Found dataset `{dataset}`.")
+
+        # Loading the model is easy. What is complicated is to get the dataset.
+        # So we start by trying to get the dataset, because if we fail, we don't
+        # want to waste time downloading the model.
+        hf_dataset = self.load_dataset(dataset, dataset_config, dataset_split, model)
+
+        # Load the model.
+        hf_model = self.load_model(model)
+
+        # Check that the dataset has the good feature names for the task.
+        feature_mapping = self._get_feature_mapping(hf_model, hf_dataset)
+
+        df = self._flatten_hf_dataset(hf_dataset, dataset_split)
+        df = pd.DataFrame(df).rename(columns={v: k for k, v in feature_mapping.items()})
+
+        # remove rows with multiple labels
+        # this is a hacky way to do it
+        # we do not support multi-label classification for now
+        if "label" in df and isinstance(df.label[0], list):
+            df = df[df.apply(lambda row: len(row['label']) == 1, axis=1)]
+        else:
+            print(df)
+        # @TODO: currently for classification models only.
+        id2label = hf_model.model.config.id2label
+        
+        if "label" in df and isinstance(df.label[0], list):
+            # need to include all labels
+            # rewrite this lambda function to include all labels
+            df.label = df.label.apply(lambda x: id2label[x[0]])
+        else:
+            # TODO: when the label for test is not provided, what do we do?
+            df["label"] = df.label.apply(lambda x: id2label[x] if x >= 0 else "-1")
+        # map the list of label ids to the list of labels
+        # df["label"] = df.label.apply(lambda x: [id2label[i] for i in x])
+        gsk_dataset = gsk.Dataset(df, target="label", column_types={"text": "text"}, validation=False)
+
+        gsk_model = HuggingFaceModel(
+            hf_model,
+            model_type="classification",
+            data_preprocessing_function=lambda df: df.text.tolist(),
+            classification_labels=[id2label[i] for i in range(len(id2label))],
+            batch_size=None,
+            device=self.device,
+        )
+
+        # Optimize batch size
+        if self.device.startswith("cuda"):
+            gsk_model.batch_size = self._find_optimal_batch_size(gsk_model, gsk_dataset)
+
+        return gsk_model, gsk_dataset
+
+    def load_dataset(self, dataset_id, dataset_config=None, dataset_split=None, model_id=None):
+        print(f"Loading dataset {dataset_id} with config {dataset_config} and split {dataset_split}")
+        """Load a dataset from the HuggingFace Hub."""
+        logger.debug(f"Trying to load dataset `{dataset_id}` (config = `{dataset_config}`, split = `{dataset_split}`).")
+        try:
+            # we do not set the split here
+            # because we want to be able to select the best split later with preprocessing
+            hf_dataset = datasets.load_dataset(dataset_id, name=dataset_config)
+            if dataset_split is None:
+                dataset_split = self._select_best_dataset_split(list(hf_dataset.keys()))
+                logger.debug(f"No split provided, automatically selected split = `{dataset_split}`).")
+                hf_dataset = hf_dataset[dataset_split]
+
+            return hf_dataset
+        except ValueError as err:
+            msg = f"Could not load dataset `{dataset_id}` with config `{dataset_config}`."
+            raise DatasetError(msg) from err
+
+    def load_model(self, model_id):
+        from transformers import pipeline
+
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+
+        return pipeline(task=task, model=model_id, device=self.device)
+
+    def _get_dataset_features(self, hf_dataset):
+        '''
+        Recursively get the features of the dataset
+        '''
+        dataset_features = {}
+        try: 
+            dataset_features = hf_dataset.features
+            return dataset_features
+        except AttributeError:
+            print("hf_dataset.features not found")
+            if isinstance(hf_dataset, datasets.DatasetDict):
+                keys = list(hf_dataset.keys())
+                return self._get_dataset_features(hf_dataset[keys[0]])
+
+    def _flatten_hf_dataset(self, hf_dataset, data_split=None):
+        '''
+        Flatten the dataset to a pandas dataframe
+        '''
+        flat_dataset = pd.DataFrame()
+        if isinstance(hf_dataset, datasets.DatasetDict):
+            keys = list(hf_dataset.keys())
+
+            for k in keys:
+                if k.startswith("train"):
+                    continue
+                elif k.startswith(data_split): 
+                    # TODO: only support one split for now
+                    # Maybe we can merge all the datasets into one
+                    flat_dataset = hf_dataset[k]
+                    break
+                else:
+                    flat_dataset = hf_dataset[k]
+            
+            # If there are only train datasets
+            if isinstance(flat_dataset, pd.DataFrame) and flat_dataset.empty:
+                flat_dataset = hf_dataset[keys[0]]
+
+        return flat_dataset
+
+    def _get_feature_mapping(self, hf_model, hf_dataset):
+        if isinstance(hf_model, TextClassificationPipeline):
+            task_features = {"text": "string", "label": "class_label"}
+        else:
+            print(type(hf_model))
+            msg = "Unsupported model type."
+            raise NotImplementedError(msg)
+
+        dataset_features = self._get_dataset_features(hf_dataset)
+        print(dataset_features)
+        # map features
+        feature_mapping = {}
+        for f in set(dataset_features):
+            if f in task_features:
+                feature_mapping[f] = f
+            else:
+                for t in task_features:
+                    if f.startswith(t):
+                        feature_mapping[t] = f
+
+        if not set(task_features) - set(feature_mapping):
+            return feature_mapping
+        else:
+            # If not, we try to find a suitable mapping by matching types.
+            return self._amend_missing_features(task_features, dataset_features, feature_mapping)
+    
+    def _amend_missing_features(self, task_features, dataset_features, feature_mapping):
+        '''
+        Question: what is this code doing?
+        '''
+        available_features = set(dataset_features) - set(feature_mapping)
+        missing_features = set(task_features) - set(feature_mapping)
+        
+        for feature in missing_features:
+            expected_type = task_features[feature]
+            if expected_type == "class_label":
+                candidates = [f for f in available_features if isinstance(dataset_features[f], datasets.ClassLabel)]
+            else:
+                candidates = [f for f in available_features if dataset_features[f].dtype == expected_type]
+
+            # If we have more than one match, it`s not possible to know which one is the good one.
+            if len(candidates) != 1:
+                msg = f"Could not find a suitable mapping for feature for `{feature}`."
+                raise RuntimeError(msg)
+
+            feature_mapping[feature] = candidates[0]
+            available_features.remove(candidates[0])
+        return feature_mapping
+
+    def _select_best_dataset_split(self, split_names):
+        """Get the best split for testing.
+
+        Selects the split `test` if available, otherwise `validation`, and as a last resort `train`.
+        If there is only one split, we return that split.
+        """
+        # If only one split is available, we just use that one.
+        if len(split_names) == 1:
+            return split_names[0]
+
+        # Otherwise iterate based on the preferred prefixes.
+        for prefix in ["test", "valid", "train"]:
+            try:
+                return next(x for x in split_names if x.startswith(prefix))
+            except StopIteration:
+                pass
+
+        return None
+
+    def _find_optimal_batch_size(self, model: BaseModel, dataset: Dataset):
+        """Find the optimal batch size for the model and dataset."""
+        initial_batch_size = model.batch_size
+        try:
+            model.batch_size = 1
+            inference_time = float("inf")
+            while True:
+                num_runs = min(30, len(dataset) // model.batch_size)
+                num_samples = num_runs * model.batch_size
+                if num_runs == 0:
+                    return model.batch_size // 2
+
+                ds_slice = dataset.slice(lambda df: df.sample(num_samples), row_level=False)
+
+                t_start = time.perf_counter_ns()
+                try:
+                    with gsk.models.cache.no_cache():
+                        model.predict(ds_slice)
+                except RuntimeError:
+                    return model.batch_size // 2
+                elapsed = time.perf_counter_ns() - t_start
+
+                time_per_sample = elapsed / (num_samples)
+                if time_per_sample > inference_time:
+                    return model.batch_size // 2
+                inference_time = time_per_sample
+                model.batch_size *= 2
+        finally:
+            model.batch_size = initial_batch_size

cicd/giskard_cicd/pipeline/runner.py

@@ -0,0 +1,48 @@
+import yaml
+import giskard as gsk
+import time
+
+
+class PipelineReport:
+    def __init__(self, scan_result):
+        self.scan_result = scan_result
+
+    def to_html(self):
+        return self.scan_result.to_html()
+
+    def to_markdown(self, template):
+        return self.scan_result.to_markdown(template="github")
+
+
+class PipelineRunner:
+    def __init__(self, loaders):
+        self.loaders = loaders
+
+    def run(self, loader_id, **kwargs):
+
+        # Get the loader
+        loader = self.loaders[loader_id]
+
+        # Get scan configuration
+        scan_config_path = kwargs.pop("scan_config", None)
+        params, detectors = None, None
+        if scan_config_path is not None:
+            with open(scan_config_path) as yaml_f:
+                scan_config = yaml.load(yaml_f, Loader=yaml.Loader)
+            params = dict(scan_config.get("configuration", None))
+            detectors = list(scan_config.get("detectors", None))
+
+        start = time.time()
+        # Load the model and dataset
+        gsk_model, gsk_dataset = loader.load_giskard_model_dataset(**kwargs)
+        print(f"Loading took {time.time() - start:.2f}s")
+
+        start = time.time()
+        # Run the scanner
+        scan_result = gsk.scan(gsk_model, gsk_dataset, params=params, only=detectors)
+        print(f"Scanning took {time.time() - start:.2f}s")
+        
+        # Report
+        report = PipelineReport(scan_result)
+
+        return report

cicd/giskard_cicd/utils.py

@@ -0,0 +1,26 @@
+import pathlib
+
+
+def dump_model_and_dataset_for_cicd(artifact_path, giskard_model, giskard_dataset):
+    from giskard.core.model_validation import validate_model, validate_model_loading_and_saving
+
+    try:
+        reloaded_model = validate_model_loading_and_saving(giskard_model)
+    except Exception as e:
+        raise Exception("An issue occured during the serialization/deserialization of your model. Please submit the traceback as a GitHub issue in the following "
+                        "repository for further assistance: https://github.com/Giskard-AI/giskard.") from e
+    try:
+        validate_model(reloaded_model, giskard_dataset)
+    except Exception as e:
+        raise Exception("An issue occured during the validation of your model. Please submit the traceback as a GitHub issue in the following "
+                        "repository for further assistance: https://github.com/Giskard-AI/giskard.") from e
+
+    pathlib.Path(artifact_path).mkdir(parents=True, exist_ok=True)
+    pathlib.Path(artifact_path+'/artifacts').mkdir(parents=True, exist_ok=True)
+    pathlib.Path(artifact_path+'/artifacts/dataset').mkdir(parents=True, exist_ok=True)
+    pathlib.Path(artifact_path+'/artifacts/model').mkdir(parents=True, exist_ok=True)
+
+    #TODO: change the Dataset.save() method to be like Model.save(), i.e. without the id requirement
+    giskard_dataset.save(pathlib.Path(artifact_path+"/artifacts/dataset"), 0)
+    giskard_model.save(pathlib.Path(artifact_path+"/artifacts/model"))
+    print("Your model and dataset are successfully dumped for CI/CD.")

cicd/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+[project]
+name = "giskard_cicd"
+readme = "README.md"
+dependencies = [
+    "datasets",
+    "giskard >= 2.0.0b",
+    "huggingface_hub",
+    "torch",
+    "transformers",
+]
+requires-python = ">=3.9"

cicd/readme.md

@@ -0,0 +1,99 @@
+# Giskard CI/CD runner (WIP)
+
+## Overview
+
+The idea is to have a common CI/CD core that can interface with different input sources (loaders) and output destinations (reporters).
+
+The **core** is responsible for running the tests and generating a report.
+
+The **loaders** are responsible for loading the model and dataset, wrapped as Giskard objects, from a given source (for example the HuggingFace hub, a Github repository, etc.).
+
+The **reporters** are responsible for sending the report to the appropriate destination (e.g. a comment to a Github PR, a HuggingFace discussion, etc.).
+
+
+### Tasks
+
+Task could be data objects containing all the information needed to run a CI/CD pipeline. For example:
+
+```json
+{
+    "loader_id": "huggingface",
+    "model": "distilbert-base-uncased",
+    "dataset": "sst2",
+    "loader_args": {
+        "dataset_split": "validation",
+    },
+    "reporter_id": "huggingface_discussion",
+    "reporter_args": {
+        "discussion_id": 1234,
+    }
+}
+```
+
+or
+
+
+```json
+{
+    "loader_id": "github",
+    "model": "my.package::load_model",
+    "dataset": "my.package::load_test_dataset",
+    "loader_args": {
+        "repository": "My-Organization/my_project",
+        "branch": "dev-test2",
+    },
+    "reporter_id": "github_pr",
+    "reported_args": {
+        "repository": "My-Organization/my_project",
+        "pr_id": 1234,
+    }
+}
+```
+
+These tasks may be generated by a watcher (e.g. a Github action, a HuggingFace webhook, etc.) and put in a queue. The CI/CD runner will then pick them up and run the pipeline.
+
+Otherwise, a single task can be created to run a single-shot Github action, without queueing.
+
+
+### CI/CD Core
+
+In pseudocode, the CI/CD core could look like this:
+
+```python
+task = get_task_from_queue_or_envirnoment()
+
+loader = get_loader(task.loader_id)
+gsk_model, gsk_dataset = loader.load_model_dataset(
+    task.model,
+    task.dataset,
+    **task.loader_args,
+)
+
+runner = PipelineRunner()
+report = runner.run(gsk_model, gsk_dataset)
+
+reporter = get_reporter(task.reporter_id)
+reporter.push_report(report, **task.reporter_args)
+```
+
+## Prototype
+
+Current implementation has two loaders:
+- The `github` loader which can be run from the command line (after running `python train.py` in `examples/github`):
+
+   ```bash
+   $ python cli.py --loader github --model examples/github/artifacts/model --dataset examples/github/artifacts/dataset
+   ```
+  
+- The `huggingface` loader which can be run from the command line:
+
+    ```bash
+    $ python cli.py --loader huggingface --model distilbert-base-uncased-finetuned-sst-2-english --dataset_split validation --output demo_report.html
+    ```
+
+- Automatically post to discussion area for a given repo
+    ```bash
+    $ python cli.py --loader huggingface --model distilbert-base-uncased-finetuned-sst-2-english --dataset_split validation --output_format markdown --output_portal huggingface --discussion_repo [REPO_ID] --hf_token [HF_TOKEN]
+    ```
+
+This will launch a pipeline that will load the model and dataset from the HuggingFace hub, run the scan and generate a report in HTML format (for now).

cicd/retriever.py

@@ -0,0 +1,55 @@
+import argparse
+import huggingface_hub
+
+
+def model_has_dataset(model):
+    for tag in model.tags:
+        if tag.startswith("dataset:"):
+            return True
+    return False
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Giskard Retriever", description="Retrieves HF models that are bound to datasets."
+    )
+    parser.add_argument(
+        "--model_type",
+        help="Hugging Face model types. default: text-classification",
+        required=False,
+    )
+    parser.add_argument("--output_format",
+                        help="Format of the information retrieved. Default: parquet. Options: parquet, csv, json.")
+
+    args = parser.parse_args()
+
+    MODEL_TYPE = args.model_type if args.model_type is not None else "text-classification"
+
+    models_with_dataset = filter(
+        model_has_dataset, huggingface_hub.list_models(filter=MODEL_TYPE, sort="likes", direction=-1)
+    )
+
+    import pandas as pd
+
+    df = pd.DataFrame(
+        [
+            {
+                "modelId": m.modelId,
+                "modelType": MODEL_TYPE,
+                "author": m.author,
+                "downloads": m.downloads,
+                "likes": m.likes,
+                "datasets": [t[8:] for t in m.tags if t.startswith("dataset:")],
+            }
+            for m in models_with_dataset
+        ]
+    )
+
+    output_format = args.output_format
+
+    if output_format is None or output_format == "parquet":
+        df.to_parquet(f"models_{MODEL_TYPE}.parquet", index=False)
+    elif output_format == "csv":
+        df.to_csv(f"models_{MODEL_TYPE}.csv", columns=df.columns, index=False)
+    elif output_format == "json":
+        df.to_json(f"models_{MODEL_TYPE}.json", index=False)

cicd/scan_config_template.yaml

@@ -0,0 +1,7 @@
+detectors:
+  - ethical_bias
+
+configuration:
+  ethical_bias:
+    threshold:
+      0.01

cicd/scan_retrieved.py

@@ -0,0 +1,107 @@
+import argparse
+import pandas as pd
+from ast import literal_eval
+from string import Template
+import os
+
+
+def model_has_dataset(model):
+    for tag in model.tags:
+        if tag.startswith("dataset:"):
+            return True
+    return False
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Giskard Batch Scanner", description="Scan Retrieved HF models."
+    )
+    parser.add_argument(
+        "--data_path",
+        help="Path to retrieved models in csv format (need to run retrieve.py first).",
+        required=True,
+    )
+    parser.add_argument("--first_Nmodels",
+                        help="Number of models to be scanned from the sorted list of models available.",
+                        required=True)
+    parser.add_argument("--output_path",
+                        help="Path of dir to save all the reports",
+                        required=True)
+
+    args = parser.parse_args()
+
+    df = pd.read_csv(args.data_path)
+
+    df_to_be_skipped = None
+    to_be_skipped_file_path = ".models_and_datasets_to_be_skipped.csv"
+    if os.path.exists(to_be_skipped_file_path):
+        df_to_be_skipped = pd.read_csv(to_be_skipped_file_path)
+
+    command_template = Template("python cli.py --loader huggingface --model $model --dataset $dataset "
+                                "--dataset_split $dataset_split --dataset_config $dataset_config "
+                                "--output ${output_path}/${model_name}__default_scan_with__${dataset_name}.html")
+
+    result_path_template = Template("${output_path}/${model_name}__default_scan_with__${dataset_name}.${suffix}")
+
+    if not os.path.exists(args.output_path):
+        os.makedirs(args.output_path)
+
+    dataset_split_exceptions = {"facebook/bart-large-mnli": "validation_matched"}
+
+    dataset_config_exceptions = {"tweet_eval": "sentiment"}
+
+    for i in range(int(args.first_Nmodels)):
+        row = df.iloc[i]
+        model = row.modelId
+        dataset = literal_eval(row.datasets)[0]
+
+        message = f"{model} with {dataset}"
+
+        if ((df_to_be_skipped['model'] == model) & (df_to_be_skipped['dataset'] == dataset)).any() \
+                and df_to_be_skipped is not None:
+            print(f"[{i}] ==== ⏩ skipping {message} ====")
+            continue
+
+        print(f"[{i}] ==== 🔍 scanning {message} ====")
+
+        result_path = result_path_template.substitute(model_name=model.replace("/", "--"),
+                                                      dataset_name=dataset.replace("/", "--"),
+                                                      output_path=args.output_path,
+                                                      suffix="html")
+        if os.path.exists(result_path):
+            answer = input(f"{result_path} already exists, Overwrite[o] or Skip[s]? ")
+
+            while answer not in ["o", "s"]:
+                answer = input("Invalid answer, please choose between 'o' and 's'")
+
+            if answer == 'o':
+                os.remove(result_path)
+            elif answer == 's':
+                continue
+
+        command = command_template.substitute(model=model, dataset=dataset,
+                                              dataset_split=dataset_split_exceptions.get(model, "validation"),
+                                              dataset_config=dataset_config_exceptions.get(dataset, None),
+                                              model_name=model.replace("/", "--"),
+                                              dataset_name=dataset.replace("/", "--"),
+                                              output_path=args.output_path)
+
+        try:
+            os.system(command)  # call the cli script in order for try, except to work
+            new_row = pd.DataFrame({"model": model, "dataset": dataset, "status": "done"}, index=[0])
+            df_to_be_skipped = pd.concat([df_to_be_skipped, new_row], ignore_index=True)
+            df_to_be_skipped.to_csv(to_be_skipped_file_path, index=False)
+        except Exception as e:
+            new_row = pd.DataFrame({"model": model, "dataset": dataset, "status": "error"}, index=[0])
+            df_to_be_skipped = pd.concat([df_to_be_skipped, new_row], ignore_index=True)
+            df_to_be_skipped.to_csv(to_be_skipped_file_path, index=False)
+            result_path = result_path_template.substitute(model_name=model.replace("/", "--"),
+                                                          dataset_name=dataset.replace("/", "--"),
+                                                          output_path=args.output_path,
+                                                          suffix="error")
+            with open(result_path, "w") as error_log:
+                error_log.write(e)
+            print(
+                f"Something went wrong while {message}, error is logged at {result_path}. "
+                "continuing with the next model...")
+            # raise Exception(f"Something went wrong while {message}") from e

cicd/setup.cfg

@@ -0,0 +1,13 @@
+[metadata]
+name = giskard_cicd
+version = 0.1.0
+
+[options]
+packages = find:
+install_requires =
+    giskard >= 2.0.0b
+    transformers
+    huggingface_hub
+    datasets
+    torch
+