scorecard_templates/financial_costs.json

{
  "name": "Financial Costs Evaluation",
  "questions": [
    {
      "question": "Comprehensive Cost Evaluation",
      "explainer": "Has a thorough assessment of the financial costs associated with the system been conducted?",
      "details": [
        "Estimation of infrastructure and hardware costs",
        "Calculation of labor hours from researchers, developers, and crowd workers",
        "Tracking of compute costs using low-cost or standard pricing per instance-hour",
        "Breakdown of costs per system component (data cost, compute cost, technical architecture)",
        "Consideration of dataset size, model size, and training volume in cost calculations"
      ]
    },
    {
      "question": "Storage and Training Cost Analysis",
      "explainer": "Have the costs for data storage and model training been evaluated?",
      "details": [
        "Assessment of storage costs for both datasets and resulting models",
        "Consideration of in-house vs. cloud storage options",
        "Evaluation of training costs based on in-house GPUs or per-hour-priced instances",
        "Analysis of cost tradeoffs considering model and dataset size",
        "Examination of memory and tier-based pricing for storage"
      ]
    },
    {
      "question": "Hosting and Inference Cost Evaluation",
      "explainer": "Have the costs associated with hosting and inference been assessed?",
      "details": [
        "Evaluation of low-latency serving costs",
        "Assessment of inference costs based on token usage",
        "Consideration of factors such as initial prompt length and requested token response length",
        "Analysis of cost variations across different languages and tokenization methods",
        "Examination of inference volume considerations and optimization for decreased latency"
      ]
    },
    {
      "question": "Modality-Specific Cost Analysis",
      "explainer": "For image, video, or audio systems, have modality-specific costs been evaluated?",
      "details": [
        "Assessment of costs related to pixel density and frame usage for image and video",
        "Evaluation of preprocessing costs for audio (e.g., spectrogram generation)",
        "Consideration of model architecture in cost calculations",
        "Analysis of inference costs specific to the modality",
        "Examination of storage and processing requirements for different media types"
      ]
    },
    {
      "question": "Long-term Cost Considerations",
      "explainer": "Does the evaluation consider long-term and indirect financial costs?",
      "details": [
        "Assessment of pre- and post-deployment costs",
        "Consideration of human labor and hidden costs",
        "Tracking of changes in costs and economy of components over time",
        "Evaluation of costs not directly tied to the system alone",
        "Analysis of potential future cost fluctuations"
      ]
    },
    {
      "question": "API Cost Evaluation",
      "explainer": "For API-accessible models, has the cost structure been evaluated?",
      "details": [
        "Assessment of token-usage based pricing",
        "Evaluation of cost variations based on initial prompt length and requested token response length",
        "Analysis of cost differences across model versions",
        "Examination of pricing structures for different types of requests",
        "Consideration of volume discounts or tiered pricing models"
      ]
    },
    {
      "question": "Comprehensive Cost Tracking",
      "explainer": "Does the evaluation attempt to track and account for both direct and indirect costs, including those not immediately tied to the system?",
      "details": [
        "Assessment of costs related to broader infrastructure or organizational changes",
        "Evaluation of long-term maintenance and update costs",
        "Analysis of costs associated with complementary technologies or processes",
        "Consideration of costs related to regulatory compliance or legal considerations"
      ]
    }
  ]
}