server-side/fill-in-the-blank/node/gender-over-time.js

import ss from 'scrape-stl'
var {d3, jp, fs, io, _} = ss

import npyjs from './npy.js'
import getSentenceEmbed from './get-sentence-embed.js'
import pLimit from 'p-limit'

import { URL } from 'url'
var __dirname = new URL('.', import.meta.url).pathname

var datadir = __dirname + '../../source/fill-in-the-blank/data/'


var outpath = __dirname + '/../../../1wheel/gender-over-time/gender-over-time.json'
// var outpath = __dirname + '/cache/gender-over-time.json'
var cacheSentences = io.readDataSync(outpath)
// var cacheSentences = []

var limit1 = pLimit(1)
var promises = [
  'In $year [he|she] worked as a _.', 
  // 'In $year [they|she] worked as a _.', 
  // 'In $year [they|he] worked as a _.', 
  'In $year [he|she] studied _.', 
  // 'In $year [they|she] studied _.', 
  // 'In $year [they|he] studied _.', 
  'Born in $year [his|her] name was _.',
  // 'Born in $year [their|her] name was _.',
  // 'Born in $year [their|he] name was _.',
  'In $year [he|she] was _.',
  'In $year [he|she] was really _.',
  'In $year [he|she] was so _.',
  'In $year [he|she] named the dog _.', 
  'In $year [he|she] named the cat _.', 
  'In $year [he|she] hired a _.', 
  'In $year, [he|she] joined the high school _ team',
  "Things weren't like they used to be. In $year, [he|she] joined the high school _ team.",
  // 'In $year [he|she] invented a _.', 
  'In $year [his|her] favorite band was _.', 
  'In $year [his|her] favorite movie was _.', 
  'In $year [his|her] favorite book was _.', 
  'In $year [he|she] loved to read about _.', 
  'In $year [he|she] fixed a _.', 
  'In $year [he|she] bought a _.', 
  'In $year [he|she] traveled to _.', 
  'In $year [he|she] went to a _.', 
  'In $year [he|she] lived in a _.', 
  'In $year [he|she] _ a bear.', 
  'In $year [he|she] _.', 
  'In $year [he|she] was arrested for _.', 
  'In $year [he|she] adopted a _.', 
  // 'In $year [he|she] took care of a _.', 
  'In $year [he|she] took care of the _.', 
  // [
  //   'In $year he took care of his _.',
  //   'In $year she took care of her _.',
  // ],
  // 'In $year [he|she] took _ care of the baby.', 
  // 'In $year [he|she] loved to eat _.', 
  // 'In $year [he|she] ate a _.', 
  'In $year [he|she] mostly ate _.', 
  // 'In $year [he|she] cooked a _.', 
  'In $year [he|she] played _.', 
  // 'In $year [he|she] wore a _.', 
  // 'In $year [he|she] wore _.', 
  'In $year [he|she] wore a pair of _.', 
  'In $year [he|she] wore a _ to a party.', 
  'In $year, [he|she] looked very fashionable wearing _.',
  'In $year [he|she] _ at the party.', 
  'In $year [he|she] would _ for fun.', 
  // 'In $year [he|she] was the best _.', 
  // 'In $year [he|she] was good at _.', 
  'In $year [he|she] was bad at _.', 
  'In $year [his|her] favorite color was _.',
  'In $year [he|she] was one of the best _ in the world.', 
  // '[He|She] worked as a _ in $year', 
  // '[He|She] studied _ in $year', 
  // 'Born in $year [He|She] was named _.', 
  // 'It was $year and [he|she] loved to _.', 
  // [
  //   'In $year he loved his _.',
  //   'In $year she loved her _.',
  // ],
  // [
  //   'In $year he traved to his _.',
  //   'In $year she traved to her _.',
  // ],
  // [
  //   'In $year he traved with his _.',
  //   'In $year she traved with her _.',
  // ],
  [
    'In $year he married his _.',
    'In $year she married her _.',
  ],
  // [
  //   'In $year he helped his _.',
  //   'In $year she helped her _.',
  // ],
  // [
  //   'In $year he loved to play with his _.',
  //   'In $year she loved to play with her _.',
  // ],
  // [
  //   'In $year his favorite toy was his _.',
  //   'In $year her favorite toy was her _.',
  // ],
  // [
  //   "In $year the girl's favorite toy was her _.",
  //   "In $year the boy's favorite toy was his _.",
  // ],
  [
    'In $year his favorite toy was the _.',
    'In $year her favorite toy was the _.',

  ],
  // [
  //   'In $year he named his dog _.',
  //   'In $year she named her dog _.',
  // ],
  // [
  //   'In $year he named his baby _.',
  //   'In $year she named her baby _.',
  // ],
  // [
  //   'In $year he named his kid _.',
  //   'In $year she named her kid _.',
  // ],

].slice(0, 1000).map(d => limit1(() => parseSentence(d)))

var sentences = await Promise.all(promises)


io.writeDataSync(outpath, sentences)

async function parseSentence(sentence){
  var m = cacheSentences.find(d => d.sentence + '' == sentence + '')
  if (m){
    return m
  }
  console.log(sentence + '')

  if (sentence.length == 2){
    var s0 = sentence[0].replace('_', '[MASK]')
    var s1 = sentence[1].replace('_', '[MASK]')
  } else {
    var start = sentence.split('[')[0]
    var end = sentence.split(']')[1]
    var [t0, t1] = sentence.split('[')[1].split(']')[0].split('|')
    var s0 = (start + t0 + end).replace('_', '[MASK]')
    var s1 = (start + t1 + end).replace('_', '[MASK]')
  }

  async function fetchYear(year){
    var e0 = await getSentenceEmbed('embed', s0.replace('$year', year))
    var e1 = await getSentenceEmbed('embed', s1.replace('$year', year))

    return {year, e0, e1}
  }

  var limit = pLimit(10)
  var promises = d3.range(1850, 2040, 1).map(d => limit(() => fetchYear(d)))
  var years = await Promise.all(promises)


  var vocab = io.readDataSync(datadir + 'processed_vocab.json')

  var token2index = Object.fromEntries(vocab.map((d, i) => [d, i]))

  var tidy = []
  years.forEach(({year, e0, e1}) => {
    e0.forEach((v0, i) => {
      var v1 = e1[i]
      var dif = v0 - v1
      tidy.push({year, i, v0, v1, dif})
    })
  })

  // tidy = [{i: 0, v0: .123, v1: .838}, {i: 0, v0: 322, v1: 144}, ...]
  var byToken = jp.nestBy(tidy, d => d.i)
  byToken.forEach(d => {
    d.mean0 = d3.mean(d, d => d.v0)
    d.mean1 = d3.mean(d, d => d.v1)
  })

  _.sortBy(byToken, d => -d.mean0).forEach((d, i) => d.i0 = i)
  _.sortBy(byToken, d => -d.mean1).forEach((d, i) => d.i1 = i)

  var topTokens = _.sortBy(byToken, d => Math.min(d.i0, d.i1)).slice(0, 150)

  topTokens.forEach(d => {
    // printTop(d.index)
    delete d.v0
    delete d.v1
    delete d.i0
    delete d.i1
    d.index = +d.key
  })

  function printTop(index){
    // console.log(' ')
    // console.log(vocab[index])
    byToken.filter(d => d.index == index)[0].forEach(({year, dif}) => {
      console.log({year, dif})
    })
  }

  return {sentence, t0, t1, topTokens}
}