summaryrefslogtreecommitdiff
path: root/cleaner.go
diff options
context:
space:
mode:
Diffstat (limited to 'cleaner.go')
-rw-r--r--cleaner.go32
1 files changed, 9 insertions, 23 deletions
diff --git a/cleaner.go b/cleaner.go
index d79e462..3d47f39 100644
--- a/cleaner.go
+++ b/cleaner.go
@@ -19,24 +19,14 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro
defer db.Close()
//Remove the processed mark on entries older than 6 hours
- err = reprocess(db, time.Now().Add(-6*time.Hour))
+ err = reprocess(db, time.Now().Add(-1*time.Hour))
if err != nil {
return
}
- var interval time.Duration
- switch conf.Interval {
- case "5min":
- interval = time.Minute * 5
- case "10min":
- interval = time.Minute * 10
- case "hour":
- interval = time.Hour
- case "day":
- interval = time.Hour * 24
- default:
- err = errors.New(fmt.Sprintf("Invalid interval: %s", conf.Interval))
- return
+ interval, err := conf.getInterval()
+ if err != nil {
+ return err
}
//Fetch data that should be cleaned
rDat, err := fetchRawData(db, time.Now().Add(-2*interval), conf.Limit)
@@ -51,11 +41,6 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro
return
}
- //Add noise for differential privacy
- for i := range cDat {
- cDat[i].occurences = diffpriv(cDat[i].occurences, 1, conf.Epsilon)
- }
-
//Begin transaction
tx, err := db.Begin()
if err != nil {
@@ -82,6 +67,7 @@ func cleanData(conf Config, db_user, db_pass, db_conn, db_name string) (err erro
}
tx.Commit()
+
return
}
@@ -120,7 +106,7 @@ func getTimespan(t time.Time, conf Config) (span time.Time, err error) {
return
}
-func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) {
+func clean(rDat []rawData, conf Config) (cDat []cleanedData, err error) {
// collect all ips so we can query for their ip blocks
ips := make(map[string]struct{})
for _, rd := range rDat {
@@ -145,7 +131,7 @@ func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) {
return
}
cDat = append(cDat,
- CleanData{
+ cleanedData{
ipbSrc: pairs[rd.ipSrc],
ipbDst: pairs[rd.ipDst],
asSrc: rd.asSrc,
@@ -163,8 +149,8 @@ func clean(rDat []RawData, conf Config) (cDat []CleanData, err error) {
return
}
-func removeDups(cDat []CleanData) []CleanData {
- ret := make([]CleanData, 0)
+func removeDups(cDat []cleanedData) []cleanedData {
+ ret := make([]cleanedData, 0)
var found bool
for ci := range cDat {
found = false