Spaces:
Running
Running
File size: 2,071 Bytes
94ed9e1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
package main
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"sync"
"time"
)
/*
Data Mining Assignment - Group 5
*/
type Symbols struct {
Symbols []string `json:"symbols"`
}
type Downloader struct {
symbols []string
}
/*
* New downloader
*/
func NewDownloader(symbols []string) *Downloader {
return &Downloader{
symbols: symbols,
}
}
/*
* Download dataset
*/
func (d *Downloader) Download(symbol string, wg *sync.WaitGroup) {
defer wg.Done()
unixTimestamp := getCurrentUnixTimestamp()
endpoint := fmt.Sprintf(
"https://query1.finance.yahoo.com/v7/finance/download/" +
"%s?period1=1410912000&period2=%s&interval=1d&events=history&includeAdjustedClose=true",
symbol, strconv.FormatInt(unixTimestamp, 10),
)
resp, err := http.Get(endpoint)
if err != nil {
log.Printf("[ERROR] error downloading %s: %v\n", symbol, err)
return
}
defer resp.Body.Close()
filename := fmt.Sprintf("./datasets/%s.csv", symbol)
file, err := os.Create(filename)
if err != nil {
log.Printf("[ERROR] error creating file for %s: %v\n", symbol, err)
return
}
defer file.Close()
_, err = io.Copy(file, resp.Body)
if err != nil {
log.Printf("[ERROR] error writing data for %s: %v\n", symbol, err)
return
}
fmt.Printf("[SUCCESS] saved: %s\n", symbol)
}
/*
* Get current UNIX timetamp
*/
func getCurrentUnixTimestamp() int64 {
now := time.Now().UTC()
return now.Unix()
}
func main() {
jsonFile, err := os.Open("./postman/symbols.json")
if err != nil {
log.Fatalf("[ERROR] failed to open JSON file: %v", err)
}
defer jsonFile.Close()
byteValue, err := ioutil.ReadAll(jsonFile)
if err != nil {
log.Fatalf("[ERROR] failed to read JSON file: %v", err)
}
var symbols Symbols
if err := json.Unmarshal(byteValue, &symbols); err != nil {
log.Fatalf("[ERROR] failed to unmarshal JSON: %v", err)
}
var wg sync.WaitGroup
downloader := NewDownloader(symbols.Symbols)
for _, symbol := range symbols.Symbols {
wg.Add(1)
go downloader.Download(symbol, &wg)
}
wg.Wait()
}
|