cleaning and updating dashboards

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76583ca11eb853af0f328a3739379afb12919cdf4751a8df4fc0d710e09ce77e
-size 2585132
+oid sha256:906e77fed8d2af6d7a4f4acf73640ce6aa3e3d4357e01c7b00af1c180f3b6eaf
+size 2460157

data/delivers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83a8e368360e97a07a7b7e1bf4b68cd4153a9cd8f2c3a37218ca2c25fa0004a7
-size 533832215
+oid sha256:2d975d5fc2c3c7c50bec7136dffa135aef50fe3802bd1650cd46f236675318af
+size 510466826

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3783e00d33252efd8cec813cd9ede010d089f984b5189b3d2e03726ea4bc16c
-size 6743682
+oid sha256:781a9625add04d8337bb3909befb6c0d4c39f3e0c9a5c47c1fb9398a96ba36ae
+size 6482896

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b080d92b95e9cb78c83a458aa9fafb1b22079c364dde7837ab89bcfd33c50481
-size 335554
+oid sha256:a2c07ed22be9463c8465d34717222f1553c5890254b350fb07aec2604e925795
+size 336922

data/requests.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c531cdd253c89a513fc52bf873fe8795e0a44ce6c033c89f07989d4f0932e424
-size 18043976
+oid sha256:bff6dbad037f5f9c3cee3631c126ba0f6e582b32b7e3fb75d79fc60a43a9dadb
+size 11547978

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4005bc0e5b1f940da6108a449f8a2a1feed75183dea917f5dcaac1f9cc64f6de
-size 40468
+oid sha256:c5db58ef61b7a256437ee87238249cd1da9d229a4ca5f9af79b02efe61fa73cc
+size 40127

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12faddc220fb5c6f9583b1029c19cb5a210ff3ae59f6fdef8a74667a7b36d53c
-size 8678809
+oid sha256:29c60bf39a93cad87fd54b7a87407cb8f50bb2cd2122e993762cec36d76e76fb
+size 8982599

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:166e8ec777aaa5fa8bfa2f62f22e753b5b2f32b2f8c4138f81282e99a0c03b69
-size 536907540
+oid sha256:cf3ed51b3e32b32e3cd9bba607afbc1e6d67583e6ca9126df68585c339964a8a
+size 378117330

scripts/get_mech_info.py

@@ -48,6 +48,8 @@ def fetch_block_number(timestamp_from: int, timestamp_to: int) -> dict:
     result_json = response.json()
     print(f"Response of the query={result_json}")
     blocks = result_json.get("data", {}).get("blocks", "")
+    if len(blocks) == 0:
+        raise ValueError(f"The query {query} did not return any results")
     return blocks[0]
 
 

scripts/tools.py

@@ -17,23 +17,18 @@
 #
 #   ------------------------------------------------------------------------------
 
-import json
 import os.path
 import re
 import time
 import random
-from dataclasses import dataclass
-from enum import Enum
 from typing import (
     Optional,
     List,
     Dict,
-    Any,
     Union,
 )
 import pandas as pd
 import requests
-from json.decoder import JSONDecodeError
 from eth_typing import ChecksumAddress
 from eth_utils import to_checksum_address
 from requests.adapters import HTTPAdapter
@@ -62,6 +57,15 @@ from utils import (
     limit_text,
     DATA_DIR,
     REQUEST_ID_FIELD,
+    MechEvent,
+    MechEventName,
+    MechRequest,
+    MechResponse,
+    EVENT_TO_MECH_STRUCT,
+    REQUEST_ID,
+    HTTP,
+    HTTPS,
+    REQUEST_SENDER,
 )
 
 CONTRACTS_PATH = "contracts"
@@ -78,13 +82,6 @@ BLOCK_DATA_NUMBER = "number"
 BLOCKS_CHUNK_SIZE = 10_000
 EVENT_ARGUMENTS = "args"
 DATA = "data"
-REQUEST_ID = "requestId"
-REQUEST_SENDER = "sender"
-PROMPT_FIELD = "prompt"
-CID_PREFIX = "f01701220"
-HTTP = "http://"
-HTTPS = HTTP[:4] + "s" + HTTP[4:]
-IPFS_ADDRESS = f"{HTTPS}gateway.autonolas.tech/ipfs/"
 IPFS_LINKS_SERIES_NAME = "ipfs_links"
 BACKOFF_FACTOR = 1
 STATUS_FORCELIST = [404, 500, 502, 503, 504]
@@ -96,7 +93,6 @@ N_IPFS_RETRIES = 1
 N_RPC_RETRIES = 100
 RPC_POLL_INTERVAL = 0.05
 IPFS_POLL_INTERVAL = 0.05
-FORMAT_UPDATE_BLOCK_NUMBER = 30411638
 IRRELEVANT_TOOLS = [
     "openai-text-davinci-002",
     "openai-text-davinci-003",
@@ -117,178 +113,6 @@ NUM_WORKERS = 10
 GET_CONTENTS_BATCH_SIZE = 1000
 
 
-class MechEventName(Enum):
-    """The mech's event names."""
-
-    REQUEST = "Request"
-    DELIVER = "Deliver"
-
-
-@dataclass
-class MechEvent:
-    """A mech's on-chain event representation."""
-
-    for_block: int
-    requestId: int
-    data: bytes
-    sender: str
-
-    def _ipfs_link(self) -> Optional[str]:
-        """Get the ipfs link for the data."""
-        return f"{IPFS_ADDRESS}{CID_PREFIX}{self.data.hex()}"
-
-    @property
-    def ipfs_request_link(self) -> Optional[str]:
-        """Get the IPFS link for the request."""
-        return f"{self._ipfs_link()}/metadata.json"
-
-    @property
-    def ipfs_deliver_link(self) -> Optional[str]:
-        """Get the IPFS link for the deliver."""
-        if self.requestId is None:
-            return None
-        return f"{self._ipfs_link()}/{self.requestId}"
-
-    def ipfs_link(self, event_name: MechEventName) -> Optional[str]:
-        """Get the ipfs link based on the event."""
-        if event_name == MechEventName.REQUEST:
-            if self.for_block < FORMAT_UPDATE_BLOCK_NUMBER:
-                return self._ipfs_link()
-            return self.ipfs_request_link
-        if event_name == MechEventName.DELIVER:
-            return self.ipfs_deliver_link
-        return None
-
-
-@dataclass(init=False)
-class MechRequest:
-    """A structure for a request to a mech."""
-
-    request_id: Optional[int]
-    request_block: Optional[int]
-    prompt_request: Optional[str]
-    tool: Optional[str]
-    nonce: Optional[str]
-    trader_address: Optional[str]
-
-    def __init__(self, **kwargs: Any) -> None:
-        """Initialize the request ignoring extra keys."""
-        self.request_id = int(kwargs.pop(REQUEST_ID, 0))
-        self.request_block = int(kwargs.pop(BLOCK_FIELD, 0))
-        self.prompt_request = kwargs.pop(PROMPT_FIELD, None)
-        self.tool = kwargs.pop("tool", None)
-        self.nonce = kwargs.pop("nonce", None)
-        self.trader_address = kwargs.pop("sender", None)
-
-
-@dataclass(init=False)
-class PredictionResponse:
-    """A response of a prediction."""
-
-    p_yes: float
-    p_no: float
-    confidence: float
-    info_utility: float
-    vote: Optional[str]
-    win_probability: Optional[float]
-
-    def __init__(self, **kwargs: Any) -> None:
-        """Initialize the mech's prediction ignoring extra keys."""
-        try:
-            self.p_yes = float(kwargs.pop("p_yes"))
-            self.p_no = float(kwargs.pop("p_no"))
-            self.confidence = float(kwargs.pop("confidence"))
-            self.info_utility = float(kwargs.pop("info_utility"))
-            self.win_probability = 0
-
-            # Validate probabilities
-            probabilities = {
-                "p_yes": self.p_yes,
-                "p_no": self.p_no,
-                "confidence": self.confidence,
-                "info_utility": self.info_utility,
-            }
-
-            for name, prob in probabilities.items():
-                if not 0 <= prob <= 1:
-                    raise ValueError(f"{name} probability is out of bounds: {prob}")
-
-            if self.p_yes + self.p_no != 1:
-                raise ValueError(
-                    f"Sum of p_yes and p_no is not 1: {self.p_yes} + {self.p_no}"
-                )
-
-            self.vote = self.get_vote()
-            self.win_probability = self.get_win_probability()
-
-        except KeyError as e:
-            raise KeyError(f"Missing key in PredictionResponse: {e}")
-        except ValueError as e:
-            raise ValueError(f"Invalid value in PredictionResponse: {e}")
-
-    def get_vote(self) -> Optional[str]:
-        """Return the vote."""
-        if self.p_no == self.p_yes:
-            return None
-        if self.p_no > self.p_yes:
-            return "No"
-        return "Yes"
-
-    def get_win_probability(self) -> Optional[float]:
-        """Return the probability estimation for winning with vote."""
-        return max(self.p_no, self.p_yes)
-
-
-@dataclass(init=False)
-class MechResponse:
-    """A structure for the response of a mech."""
-
-    request_id: int
-    deliver_block: Optional[int]
-    result: Optional[PredictionResponse]
-    error: Optional[str]
-    error_message: Optional[str]
-    prompt_response: Optional[str]
-    mech_address: Optional[str]
-
-    def __init__(self, **kwargs: Any) -> None:
-        """Initialize the mech's response ignoring extra keys."""
-        self.error = kwargs.get("error", None)
-        self.request_id = int(kwargs.get(REQUEST_ID, 0))
-        self.deliver_block = int(kwargs.get(BLOCK_FIELD, 0))
-        self.result = kwargs.get("result", None)
-        self.prompt_response = kwargs.get(PROMPT_FIELD, None)
-        self.mech_address = kwargs.get("sender", None)
-
-        if self.result != "Invalid response":
-            self.error_message = kwargs.get("error_message", None)
-
-            try:
-                if isinstance(self.result, str):
-                    kwargs = json.loads(self.result)
-                    self.result = PredictionResponse(**kwargs)
-                    self.error = 0
-
-            except JSONDecodeError:
-                self.error_message = "Response parsing error"
-                self.error = 1
-
-            except Exception as e:
-                self.error_message = str(e)
-                self.error = 1
-
-        else:
-            self.error_message = "Invalid response from tool"
-            self.error = 1
-            self.result = None
-
-
-EVENT_TO_MECH_STRUCT = {
-    MechEventName.REQUEST: MechRequest,
-    MechEventName.DELIVER: MechResponse,
-}
-
-
 def get_events(
     w3: Web3,
     event: str,
@@ -554,7 +378,8 @@ def etl(
     rpcs: List[str],
     filename: Optional[str] = None,
 ) -> pd.DataFrame:
-    """Fetch from on-chain events, process, store and return the tools' results on all the questions as a Dataframe."""
+    """Fetch from on-chain events, process, store and return the tools' results on
+    all the questions as a Dataframe."""
     w3s = [Web3(HTTPProvider(r)) for r in rpcs]
     session = create_session()
     event_to_transformer = {

scripts/utils.py

@@ -1,12 +1,15 @@
 import sys
+import json
 import os
 import time
 from tqdm import tqdm
-from tools import MechEventName
-from typing import List
+from typing import List, Any, Optional
 import pandas as pd
 import gc
+from dataclasses import dataclass
 from pathlib import Path
+from enum import Enum
+from json.decoder import JSONDecodeError
 
 REDUCE_FACTOR = 0.25
 SLEEP = 0.5
@@ -15,6 +18,186 @@ SCRIPTS_DIR = Path(__file__).parent
 ROOT_DIR = SCRIPTS_DIR.parent
 DATA_DIR = ROOT_DIR / "data"
 BLOCK_FIELD = "block"
+CID_PREFIX = "f01701220"
+REQUEST_ID = "requestId"
+REQUEST_SENDER = "sender"
+PROMPT_FIELD = "prompt"
+HTTP = "http://"
+HTTPS = HTTP[:4] + "s" + HTTP[4:]
+IPFS_ADDRESS = f"{HTTPS}gateway.autonolas.tech/ipfs/"
+FORMAT_UPDATE_BLOCK_NUMBER = 30411638
+
+
+class MechEventName(Enum):
+    """The mech's event names."""
+
+    REQUEST = "Request"
+    DELIVER = "Deliver"
+
+
+@dataclass
+class MechEvent:
+    """A mech's on-chain event representation."""
+
+    for_block: int
+    requestId: int
+    data: bytes
+    sender: str
+
+    def _ipfs_link(self) -> Optional[str]:
+        """Get the ipfs link for the data."""
+        return f"{IPFS_ADDRESS}{CID_PREFIX}{self.data.hex()}"
+
+    @property
+    def ipfs_request_link(self) -> Optional[str]:
+        """Get the IPFS link for the request."""
+        return f"{self._ipfs_link()}/metadata.json"
+
+    @property
+    def ipfs_deliver_link(self) -> Optional[str]:
+        """Get the IPFS link for the deliver."""
+        if self.requestId is None:
+            return None
+        return f"{self._ipfs_link()}/{self.requestId}"
+
+    def ipfs_link(self, event_name: MechEventName) -> Optional[str]:
+        """Get the ipfs link based on the event."""
+        if event_name == MechEventName.REQUEST:
+            if self.for_block < FORMAT_UPDATE_BLOCK_NUMBER:
+                return self._ipfs_link()
+            return self.ipfs_request_link
+        if event_name == MechEventName.DELIVER:
+            return self.ipfs_deliver_link
+        return None
+
+
+@dataclass(init=False)
+class MechRequest:
+    """A structure for a request to a mech."""
+
+    request_id: Optional[int]
+    request_block: Optional[int]
+    prompt_request: Optional[str]
+    tool: Optional[str]
+    nonce: Optional[str]
+    trader_address: Optional[str]
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize the request ignoring extra keys."""
+        self.request_id = int(kwargs.pop(REQUEST_ID, 0))
+        self.request_block = int(kwargs.pop(BLOCK_FIELD, 0))
+        self.prompt_request = kwargs.pop(PROMPT_FIELD, None)
+        self.tool = kwargs.pop("tool", None)
+        self.nonce = kwargs.pop("nonce", None)
+        self.trader_address = kwargs.pop("sender", None)
+
+
+@dataclass(init=False)
+class PredictionResponse:
+    """A response of a prediction."""
+
+    p_yes: float
+    p_no: float
+    confidence: float
+    info_utility: float
+    vote: Optional[str]
+    win_probability: Optional[float]
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize the mech's prediction ignoring extra keys."""
+        try:
+            self.p_yes = float(kwargs.pop("p_yes"))
+            self.p_no = float(kwargs.pop("p_no"))
+            self.confidence = float(kwargs.pop("confidence"))
+            self.info_utility = float(kwargs.pop("info_utility"))
+            self.win_probability = 0
+
+            # Validate probabilities
+            probabilities = {
+                "p_yes": self.p_yes,
+                "p_no": self.p_no,
+                "confidence": self.confidence,
+                "info_utility": self.info_utility,
+            }
+
+            for name, prob in probabilities.items():
+                if not 0 <= prob <= 1:
+                    raise ValueError(f"{name} probability is out of bounds: {prob}")
+
+            if self.p_yes + self.p_no != 1:
+                raise ValueError(
+                    f"Sum of p_yes and p_no is not 1: {self.p_yes} + {self.p_no}"
+                )
+
+            self.vote = self.get_vote()
+            self.win_probability = self.get_win_probability()
+
+        except KeyError as e:
+            raise KeyError(f"Missing key in PredictionResponse: {e}")
+        except ValueError as e:
+            raise ValueError(f"Invalid value in PredictionResponse: {e}")
+
+    def get_vote(self) -> Optional[str]:
+        """Return the vote."""
+        if self.p_no == self.p_yes:
+            return None
+        if self.p_no > self.p_yes:
+            return "No"
+        return "Yes"
+
+    def get_win_probability(self) -> Optional[float]:
+        """Return the probability estimation for winning with vote."""
+        return max(self.p_no, self.p_yes)
+
+
+@dataclass(init=False)
+class MechResponse:
+    """A structure for the response of a mech."""
+
+    request_id: int
+    deliver_block: Optional[int]
+    result: Optional[PredictionResponse]
+    error: Optional[str]
+    error_message: Optional[str]
+    prompt_response: Optional[str]
+    mech_address: Optional[str]
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize the mech's response ignoring extra keys."""
+        self.error = kwargs.get("error", None)
+        self.request_id = int(kwargs.get(REQUEST_ID, 0))
+        self.deliver_block = int(kwargs.get(BLOCK_FIELD, 0))
+        self.result = kwargs.get("result", None)
+        self.prompt_response = kwargs.get(PROMPT_FIELD, None)
+        self.mech_address = kwargs.get("sender", None)
+
+        if self.result != "Invalid response":
+            self.error_message = kwargs.get("error_message", None)
+
+            try:
+                if isinstance(self.result, str):
+                    kwargs = json.loads(self.result)
+                    self.result = PredictionResponse(**kwargs)
+                    self.error = 0
+
+            except JSONDecodeError:
+                self.error_message = "Response parsing error"
+                self.error = 1
+
+            except Exception as e:
+                self.error_message = str(e)
+                self.error = 1
+
+        else:
+            self.error_message = "Invalid response from tool"
+            self.error = 1
+            self.result = None
+
+
+EVENT_TO_MECH_STRUCT = {
+    MechEventName.REQUEST: MechRequest,
+    MechEventName.DELIVER: MechResponse,
+}
 
 
 def parse_args() -> str: