Contact
CoCalc Logo Icon
StoreFeaturesDocsShareSupport News AboutSign UpSign In
| Download
Views: 39383
1
###############################################################################
2
#
3
# CoCalc: Collaborative Calculation in the Cloud
4
#
5
# Copyright (C) 2016, Sagemath Inc.
6
#
7
# This program is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU General Public License as published by
9
# the Free Software Foundation, either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# This program is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
#
20
###############################################################################
21
22
23
"""
24
Miscellaneous functions.
25
"""
26
27
import os, sha
28
29
##########################################################################
30
# Connecting to HTTP url's.
31
##########################################################################
32
import socket, urllib, urllib2
33
34
def get(url, data=None, timeout=10):
35
old_timeout = socket.getdefaulttimeout()
36
socket.setdefaulttimeout(timeout)
37
try:
38
if data is not None:
39
url = url + '?' + urllib.urlencode(data)
40
return urllib2.urlopen(url).read()
41
finally:
42
socket.setdefaulttimeout(old_timeout)
43
44
def post(url, data=None, timeout=10):
45
old_timeout = socket.getdefaulttimeout()
46
socket.setdefaulttimeout(timeout)
47
try:
48
if data is None: data = {}
49
data = urllib.urlencode(data)
50
req = urllib2.Request(url, data)
51
response = urllib2.urlopen(req)
52
return response.read()
53
finally:
54
socket.setdefaulttimeout(old_timeout)
55
56
##########################################################################
57
# Misc file/path management functions
58
##########################################################################
59
def all_files(path): # TODO: delete if not used
60
"""
61
Return a sorted list of the names of all files in the given path, and in
62
all subdirectories. Empty directories are ignored.
63
64
INPUT:
65
66
- ``path`` -- string
67
68
EXAMPLES::
69
70
We create 3 files: a, xyz.abc, and m/n/k/foo. We also create a
71
directory x/y/z, which is empty::
72
73
>>> import tempfile
74
>>> d = tempfile.mkdtemp()
75
>>> o = open(os.path.join(d,'a'),'w')
76
>>> o = open(os.path.join(d,'xyz.abc'),'w')
77
>>> os.makedirs(os.path.join(d, 'x', 'y', 'z'))
78
>>> os.makedirs(os.path.join(d, 'm', 'n', 'k'))
79
>>> o = open(os.path.join(d,'m', 'n', 'k', 'foo'),'w')
80
81
This all_files function returns a list of the 3 files, but
82
completely ignores the empty directory::
83
84
>>> all_files(d) # ... = / on unix but \\ windows
85
['a', 'm...n...k...foo', 'xyz.abc']
86
>>> import shutil; shutil.rmtree(d) # clean up mess
87
"""
88
all = []
89
n = len(path)
90
for root, dirs, files in os.walk(path):
91
for fname in files:
92
all.append(os.path.join(root[n+1:], fname))
93
all.sort()
94
return all
95
96
97
import tempfile
98
99
_temp_prefix = None
100
def is_temp_directory(path): # TODO: delete if not used
101
"""
102
Return True if the given path is likely to have been
103
generated by the tempfile.mktemp function.
104
105
EXAMPLES::
106
107
>>> import tempfile
108
>>> is_temp_directory(tempfile.mktemp())
109
True
110
>>> is_temp_directory(tempfile.mktemp() + '../..')
111
False
112
"""
113
global _temp_prefix
114
if _temp_prefix is None:
115
_temp_prefix = os.path.split(tempfile.mktemp())[0]
116
path = os.path.split(os.path.abspath(path))[0]
117
return os.path.samefile(_temp_prefix, path)
118
119
120
##########################################################################
121
# Misc process functions
122
##########################################################################
123
124
def is_running(pid):
125
"""Return True only if the process with given pid is running."""
126
try:
127
os.kill(pid,0)
128
return True
129
except:
130
return False
131
132
133
##########################################################################
134
# Misc misc network stuff
135
##########################################################################
136
137
def local_ip_address(dest='8.8.8.8'):
138
"""
139
Return the ip address of the local network interface that is used
140
to communicate with the given destination address.
141
The default dest is 8.8.8.8, which is google's DNS.
142
"""
143
# See http://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib
144
# Obviously, this requires internet access.
145
import socket
146
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
147
s.connect((dest,80))
148
return s.getsockname()[0]
149
150
##########################################################################
151
# Other
152
##########################################################################
153
154
import random, time, traceback
155
156
class call_until_succeed(object):
157
"""
158
Decorator that makes it so that whenever a function is called, if
159
any exception is raised, then the function is re-called with
160
exactly the same arguments, after a delay. The delay starts at
161
mindelay seconds and exponentially (times 2) increases (with a
162
slight random factor) until hitting maxdelay. If it delays for a
163
total of totaldelay and totaldelay is not None, then it gives up
164
and re-raises the last exception.
165
"""
166
def __init__(self, mindelay, maxdelay, totaldelay=None):
167
self._mindelay = mindelay
168
self._maxdelay = maxdelay
169
self._totaldelay = totaldelay
170
171
def __call__(self, f):
172
def g(*args, **kwds):
173
attempts = 0
174
delay = (random.random() + 1)*self._mindelay
175
totaldelay = 0
176
while True:
177
attempts += 1
178
try:
179
return f(*args, **kwds)
180
except:
181
if self._totaldelay and totaldelay >= self._totaldelay:
182
print("call_until_succeed: exception hit running %s; too long (>=%s)"%(f.__name__, self._totaldelay))
183
raise
184
185
# print stack trace and name of function
186
print("call_until_succeed: exception hit %s times in a row running %s; retrying in %s seconds"%(attempts, f.__name__, delay))
187
traceback.print_exc()
188
time.sleep(delay)
189
totaldelay += delay
190
delay = min(2*delay, self._maxdelay)
191
192
return g
193
194
195
##########################################################################
196
# Parallel computing
197
##########################################################################
198
199
def thread_map(callable, inputs):
200
"""
201
Computing [callable(*args, **kwds) for (args,kwds) in inputs] in parallel using
202
len(inputs) separate threads.
203
204
If an exception is raised by any thread, a RuntimeError exception
205
is instead raised.
206
"""
207
from threading import Thread
208
class F(Thread):
209
def __init__(self, x):
210
self._x = x
211
Thread.__init__(self)
212
self.start()
213
def run(self):
214
try:
215
self.result = callable(*self._x[0], **self._x[1])
216
self.fail = False
217
except Exception, msg:
218
self.result = msg
219
self.fail = True
220
results = [F(x) for x in inputs]
221
for f in results: f.join()
222
e = [f.result for f in results if f.fail]
223
if e: raise RuntimeError(e)
224
return [f.result for f in results]
225
226
227
################################################
228
# sha-1 hash
229
################################################
230
def sha1(s):
231
return sha.sha(s).hexdigest()
232
233