diff options
Diffstat (limited to 'cleaner.go')
| -rw-r--r-- | cleaner.go | 32 |
1 files changed, 9 insertions, 23 deletions
@@ -19,24 +19,14 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro defer db.Close() //Remove the processed mark on entries older than 6 hours - err = reprocess(db, time.Now().Add(-6*time.Hour)) + err = reprocess(db, time.Now().Add(-1*time.Hour)) if err != nil { return } - var interval time.Duration - switch conf.Interval { - case "5min": - interval = time.Minute * 5 - case "10min": - interval = time.Minute * 10 - case "hour": - interval = time.Hour - case "day": - interval = time.Hour * 24 - default: - err = errors.New(fmt.Sprintf("Invalid interval: %s", conf.Interval)) - return + interval, err := conf.getInterval() + if err != nil { + return err } //Fetch data that should be cleaned rDat, err := fetchRawData(db, time.Now().Add(-2*interval), conf.Limit) @@ -51,11 +41,6 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro return } - //Add noise for differential privacy - for i := range cDat { - cDat[i].occurences = diffpriv(cDat[i].occurences, 1, conf.Epsilon) - } - //Begin transaction tx, err := db.Begin() if err != nil { @@ -82,6 +67,7 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro } tx.Commit() + return } @@ -120,7 +106,7 @@ func getTimespan(t time.Time, conf Config) (span time.Time, err error) { return } -func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) { +func clean(rDat []rawData, conf Config) (cDat []cleanedData, err error) { // collect all ips so we can query for their ip blocks ips := make(map[string]struct{}) for _, rd := range rDat { @@ -145,7 +131,7 @@ func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) { return } cDat = append(cDat, - CleanData{ + cleanedData{ ipbSrc: pairs[rd.ipSrc], ipbDst: pairs[rd.ipDst], asSrc: rd.asSrc, @@ -163,8 +149,8 @@ func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) { return } -func removeDups(cDat []CleanData) []CleanData { - ret := make([]CleanData, 0) +func removeDups(cDat []cleanedData) []cleanedData { + ret := make([]cleanedData, 0) var found bool for ci := range cDat { found = false |