CoCalc Shared Fileswww / cgi-bin / mfd / mfd_compression.pyOpen in CoCalc with one click!
Author: William A. Stein
1
import misc
2
import constants
3
import magma, pari
4
import pg # postgresql access
5
import mfd_opendb
6
import os, string
7
import heegner
8
import mfe_source
9
import sys
10
import bzip2
11
12
db = mfd_opendb.opendb()
13
14
DATA = constants.home + '/data'
15
UNKNOWN = constants.not_computed
16
ESCAPE = constants.bin + '/escape_string'
17
UNESCAPE = constants.bin + '/unescape_string'
18
FLUSH_FREQ = constants.flush_freq
19
BUFF_LIMIT = constants.buff_limit
20
COMPRESS_LIMIT = constants.compress_limit
21
22
##########################################
23
########### Escaping Routines ############
24
##########################################
25
def escape_string(e):
26
# open a pipe
27
X = os.popen2(ESCAPE + ' %s %s'%(len(e), FLUSH_FREQ))
28
escape_in = X[0]
29
escape_out = X[1]
30
emb = ""
31
buff_pt = 0
32
while buff_pt < len(e):
33
buff = e[buff_pt : buff_pt + BUFF_LIMIT]
34
escape_in.write(buff)
35
escape_in.flush()
36
emb = emb + escape_out.read(5*len(buff))
37
buff_pt = buff_pt + BUFF_LIMIT
38
return emb
39
40
# uses temporary files - otherwise it is too messy
41
def unescape_string(st):
42
# create filenames
43
finname = misc.newtempfilename();
44
foutname = misc.newtempfilename();
45
46
# write string into a temporary file
47
fin = open(finname, "w")
48
fin.write(st)
49
fin.close()
50
51
# a system call to the unescape_string C program
52
cmd = UNESCAPE + " " + "%s"%len(st) + " < " + finname + " > " + foutname
53
os.system(cmd)
54
fout = open(foutname, "r")
55
ans = fout.read()
56
fout.close()
57
58
# remove the files
59
os.system("rm %s %s;"%(finname, foutname))
60
return ans
61
#####################################################
62
############### End Escaping Routines ###############
63
#####################################################
64
65
66
67
#####################################################
68
############### START: ap --> value #################
69
#####################################################
70
71
def set_ap_compress(mod_id, p, ap):
72
query = "DELETE FROM ap WHERE mod_id=%s AND p=%s"%(mod_id, p)
73
db.query(query)
74
75
# check whether ap value is compressed
76
if len(ap) > COMPRESS_LIMIT:
77
comp_ap = bzip2.compress(ap)
78
escaped_ap = escape_string(comp_ap)
79
query = "INSERT INTO ap(mod_id, p, compressed_value) VALUES(%s, %s, '%s')"%\
80
(mod_id, p, escaped_ap)
81
else:
82
query = "INSERT INTO ap(mod_id, p, value) VALUES(%s, %s, '%s')"%\
83
(mod_id, p, ap)
84
db.query(query)
85
86
def select_ap(mod_id, p):
87
query = "SELECT compressed_value FROM ap WHERE mod_id=%s AND p=%s AND compressed_value IS NOT NULL"%(mod_id, p)
88
out = db.query.dictresult()
89
90
if len(out) > 0:
91
escaped_ap = out[0]['compressed_value']
92
return bzip2.decompress(unescape_string(escaped_ap))
93
94
query = "SELECT value FROM ap WHERE mod_id=%s AND p=%s AND value IS NOT NULL"%(mod_id, p)
95
out = db.query.dictresult()
96
97
if len(out) > 0:
98
return out[0]['value']
99
100
return constants.not_computed
101
102
###################################################
103
############### END: ap --> value #################
104
##################################################
105
106
107
108
###############################################################
109
############### START: ap_data --> embedding ##################
110
###############################################################
111
112
def set_ap_data_compress(mod_id, field, embedding, source):
113
query = "DELETE FROM ap_data WHERE mod_id=%s"%mod_id
114
db.query(query)
115
116
if len(embedding) > COMPRESS_LIMIT:
117
e = bzip2.compress(embedding)
118
emb = escape_string(e)
119
query = "INSERT INTO ap_data(mod_id, field, compressed_embedding) VALUES(%s, '%s', '%s')"%\
120
(mod_id, field, emb)
121
else:
122
query = "INSERT INTO ap_data(mod_id, field, embedding) VALUES(%s, '%s', '%s')"%\
123
(mod_id, field, embedding)
124
db.query(query)
125
126
if source:
127
query = "UPDATE ap_data SET source='%s' WHERE mod_id=%s"%(source, mod_id)
128
db.query(query)
129
130
def select_ap_embedding(mod_id):
131
query = "SELECT compressed_embedding FROM ap_data WHERE mod_id=%s AND compressed_embedding IS NOT NULL"%mod_id
132
out = db.query(query).dictresult()
133
134
if len(out) > 0:
135
escaped_embedding = out[0]['compressed_embedding']
136
return bzip2.decompress(unescape_string(escaped_embedding))
137
138
query = "SELECT embedding FROM ap_data WHERE mod_id=%s AND embedding IS NOT NULL"%mod_id
139
out = db.query(query).dictresult()
140
141
if len(out) > 0:
142
return out[0]['embedding']
143
144
return constants.not_computed
145
146
#######################################################
147
############# END: ap_data --> embedding ##############
148
#######################################################
149
150
151
152
#######################################################
153
############# charpoly_2 --> charpoly #################
154
#######################################################
155
def set_charpoly_compressed(mod_id, n, charpoly, source):
156
query = "DELETE FROM charpoly_2 WHERE mod_id=%s AND n=%s"%(mod_id, n)
157
db.query(query)
158
159
# check whether embedding should be compressed
160
if len(charpoly) > COMPRESS_LIMIT:
161
comp_charpoly = bzip2.compress(charpoly)
162
escaped_charpoly = escape_string(comp_charpoly)
163
query = "INSERT INTO charpoly_2(mod_id, n, compressed_charpoly) VALUES(%s, %s, '%s')"%\
164
(mod_id, n, escaped_charpoly)
165
else:
166
query = "INSERT INTO charpoly_2(mod_id, n, charpoly) VALUES(%s, %s, '%s')"%\
167
(mod_id, n, charpoly)
168
db.query(query)
169
170
if source:
171
query = "UPDATE charpoly_2 SET source='%s' WHERE mod_id=%s AND n=%s"%(source, mod_id, n)
172
db.query(query)
173
174
def select_charpoly_compressed(mod_id, n):
175
query = "SELECT compressed_charpoly FROM charpoly_2 WHERE mod_id=%s AND n=%s "%(mod_id, n) +\
176
"AND compressed_charpoly IS NOT NULL"
177
178
out = db.query(query).dictresult()
179
180
if len(out) > 0:
181
escaped_charpoly = out[0]['compressed_charpoly']
182
return bzip2.decompress(unescape_string(escaped_charpoly))
183
184
query = "SELECT charpoly FROM charpoly_2 WHERE mod_id=%s AND n=%s AND charpoly IS NOT NULL"%(mod_id, n)
185
out = db.query(query).dictresult()
186
if len(out) > 0:
187
return out[0]['charpoly']
188
189
return constants.not_computed
190
191
#######################################################
192
############# END: charpoly_2 --> charpoly ############
193
#######################################################
194