from datasets import load_dataset
import datasets
import json
import numpy as np
import os
#Dataset loading script that is missing quite a lot of details but works
class NewDataset(datasets.GeneratorBasedBuilder):
    def _info(self):
        return datasets.DatasetInfo(
            description="beep boop",
            features=datasets.Features(
                {
                    "description": datasets.Value("string"),
                    "text": datasets.Value("string"),
                    "rating": datasets.Value("int32")
                }
            ),
            # No default supervised_keys (as we have to pass both question
            # and context as input).
            supervised_keys=None,
            homepage="no",
            citation="no",
        )
        
    def _split_generators(self, dl_manager):
            files = self.config.data_files
            return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"files": files["train"]})]
    def _generate_examples(
        self, files  # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
    ):
        """ Yields examples as (key, example) tuples. """
        # This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
        # The `key` is here for legacy reason (tfds) and is not important in itself.
        #print("files",files)
        key = 0
        for file in files:
            with open(file, encoding="utf-8") as f:
                data = json.load(f)

                for item in data:
                    for review in item["reviews"]:
                        yield key, {
                            "description": item["description_raw"],
                            "text": review["reviewText"],
                            "rating": review["rating"],
                        }
                        key += 1