CoCalc Public Filesha-ram-anonymous-data / demo.pyOpen with one click!
Authors: Harald Schilly, Harald Schilly, ℏal Snyder, William A. Stein
Views : 63
Compute Environment: Ubuntu 18.04 (Deprecated)
1
#!/usr/bin/env python
2
# demo about how to read the data files
3
4
import csv
5
import gzip
6
from datetime import datetime
7
from dateutil.parser import parse as dt_parse
8
9
# february data
10
fn = 'smc_file_access_2016-02.csv.gz'
11
file_handler = csv.reader(gzip.open(fn), delimiter=";")
12
13
# the following demo computes the distinct active users
14
# by weekday and distinct projects per day of month
15
16
proj_per_day = {}
17
account_per_weekday = {}
18
19
for line_no, row in enumerate(file_handler):
20
if line_no == 0:
21
# print "HEADER: %s" % row
22
continue
23
24
if line_no % 100000 == 0:
25
print "at line %d" % line_no
26
# break # uncomment, to break early while developing
27
28
# parse rows entries
29
timestamp = dt_parse(row[0])
30
account = int(row[1])
31
project = int(row[2])
32
filename = int(row[3])
33
extension = row[4]
34
35
# derived information
36
weekday = timestamp.weekday() # 0 to 6
37
hour = timestamp.hour
38
day = timestamp.day
39
40
proj_per_day.setdefault(day, set())
41
proj_per_day[day].add(project)
42
43
account_per_weekday.setdefault(weekday, set())
44
account_per_weekday[weekday].add(account)
45
46
print
47
for day, proj in sorted(proj_per_day.iteritems()):
48
print "Day %3d: %5d projects active" % (day, len(proj))
49
print
50
51
for weekday, accounts in sorted(account_per_weekday.iteritems()):
52
print "Weekday %2d: %5d accounts active" % (weekday, len(accounts))
53
print