Spaces:

HuggingFaceFW
/

blogpost-fineweb-v1

Running

hynky HF staff commited on Jun 1

Commit

01fd83e

•

1 Parent(s): 9f13c43

gziped

Files changed (6) hide show

dist/bundle.js.gz ADDED Viewed

Binary file (429 kB). View file

dist/index.html CHANGED Viewed

@@ -1,4 +1,4 @@
-<!doctype html><head><meta name="viewport" content="width=device-width,initial-scale=1"><meta charset="utf8"><base target="_blank"><title>FineWeb: decanting the web for the finest text data at scale</title><link rel="stylesheet" href="style.css"><style>/* ****************************************
          * TOC
          ******************************************/
         @media (max-width: 1199px) {
@@ -98,7 +98,7 @@
         d-contents nav > div > a:hover,
         d-contents nav > ul > li > a:hover {
             text-decoration: none;
-        }</style><script defer="defer" src="bundle.js"></script></head><body><d-front-matter><script id="distill-front-matter" type="text/json">{
     "title": "🍷 FineWeb: decanting the web for the finest text data at scale",
     "description": "This blog covers a discussion on processing and evaluating data quality at scale, the 🍷 FineWeb recipe (listing and explaining all of our design choices), and the process followed to create 📚 FineWeb-Edu.",
     "published": "May 28, 2024",

+<!doctype html><head><link rel="stylesheet" href="style.css"><meta name="viewport" content="width=device-width,initial-scale=1"><meta charset="utf8"><base target="_blank"><title>FineWeb: decanting the web for the finest text data at scale</title><style>/* ****************************************
          * TOC
          ******************************************/
         @media (max-width: 1199px) {
         d-contents nav > div > a:hover,
         d-contents nav > ul > li > a:hover {
             text-decoration: none;
+        }</style><script defer="defer" src="bundle.js.gz"></script></head><body><d-front-matter><script id="distill-front-matter" type="text/json">{
     "title": "🍷 FineWeb: decanting the web for the finest text data at scale",
     "description": "This blog covers a discussion on processing and evaluating data quality at scale, the 🍷 FineWeb recipe (listing and explaining all of our design choices), and the process followed to create 📚 FineWeb-Edu.",
     "published": "May 28, 2024",

package-lock.json ADDED Viewed

The diff for this file is too large to render. See raw diff

package.json CHANGED Viewed

@@ -18,8 +18,10 @@
     "@babel/preset-env": "^7.24.6",
     "babel-loader": "^9.1.3",
     "clean-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^12.0.2",
     "css-loader": "^7.1.2",
     "html-webpack-plugin": "^5.6.0",
     "style-loader": "^4.0.0",
     "webpack": "^5.91.0",

     "@babel/preset-env": "^7.24.6",
     "babel-loader": "^9.1.3",
     "clean-webpack-plugin": "^4.0.0",
+    "compression-webpack-plugin": "^11.1.0",
     "copy-webpack-plugin": "^12.0.2",
     "css-loader": "^7.1.2",
+    "html-webpack-change-assets-extension-plugin": "^1.3.1",
     "html-webpack-plugin": "^5.6.0",
     "style-loader": "^4.0.0",
     "webpack": "^5.91.0",

src/index.html CHANGED Viewed

@@ -1,12 +1,11 @@
 <!doctype html>
 <head>
-    <!-- <link rel="stylesheet" href="style.css"> -->
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta charset="utf8">
     <base target="_blank">
     <title>FineWeb: decanting the web for the finest text data at scale</title>
-    <link rel="stylesheet" href="style.css">
     <style>
         /* ****************************************

 <!doctype html>
 <head>
+    <link rel="stylesheet" href="style.css">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta charset="utf8">
     <base target="_blank">
     <title>FineWeb: decanting the web for the finest text data at scale</title>
     <style>
         /* ****************************************

webpack.config.js CHANGED Viewed

@@ -2,7 +2,8 @@ const path = require("path");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const CopyPlugin = require("copy-webpack-plugin");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
-const ChangeDataColors = require("./changeDataColors.js");
 const COLOR_KEYS = ["color", "bgColor", "fillcolor"];
@@ -60,8 +61,12 @@ module.exports = {
         new CleanWebpackPlugin(),
         new HtmlWebpackPlugin({
             template: "./src/index.html", // Path to your source template
-            filename: "index.html", // Optional: specify a filename (defaults to index.html)
         }),
         new CopyPlugin({
             patterns: [
                 {
@@ -82,3 +87,5 @@ module.exports = {
     },
     mode: process.env.NODE_ENV === 'production' ? 'production' : 'development',
 };

 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const CopyPlugin = require("copy-webpack-plugin");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const CompressionPlugin = require("compression-webpack-plugin");
+const HtmlWebpackChangeAssetsExtensionPlugin = require("html-webpack-change-assets-extension-plugin");
 const COLOR_KEYS = ["color", "bgColor", "fillcolor"];
         new CleanWebpackPlugin(),
         new HtmlWebpackPlugin({
             template: "./src/index.html", // Path to your source template
+            jsExtension: ".gz"
         }),
+        new CompressionPlugin({
+            test: /\.(js|mjs)(\?.*)?$/i,
+        }),
+        new HtmlWebpackChangeAssetsExtensionPlugin(),
         new CopyPlugin({
             patterns: [
                 {
     },
     mode: process.env.NODE_ENV === 'production' ? 'production' : 'development',
 };
+console.log(process.env.NODE_ENV)