Python分布式计算（作品）

Question

我正在使用旧线程发布试图解决相同问题的新代码。 什么构成安全的泡菜？ this?

sock.py

from socket import socket 
from socket import AF_INET 
from socket import SOCK_STREAM 
from socket import gethostbyname 
from socket import gethostname 

class SocketServer: 
    def __init__(self, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.port = port 
    def listen(self, data): 
    self.sock.bind(("127.0.0.1", self.port)) 
    self.sock.listen(len(data)) 
    while data: 
     s = self.sock.accept()[0] 
     siz, dat = data.pop() 
     s.send(siz) 
     s.send(dat) 
     s.close() 

class Socket: 
    def __init__(self, host, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.sock.connect((host, port)) 
    def recv(self, size): 
    return self.sock.recv(size) 

pack.py

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 
from marshal import dumps as marshal_dumps 
from pickle import dumps as pickle_dumps 
from struct import pack as struct_pack 

class packer: 
    def __init__(self): 
    self.f = [] 
    def pack(self, what): 
    if type(what) is type(lambda:None): 
     self.f = [] 
     self.f.append(marshal_dumps(what.func_code)) 
     self.f.append(pickle_dumps(what.func_name)) 
     self.f.append(pickle_dumps(what.func_defaults)) 
     self.f.append(pickle_dumps(what.func_closure)) 
     self.f = pickle_dumps(self.f) 
     return (struct_pack('Q', len(self.f)), self.f) 

unpack.py

from types import FunctionType 
from pickle import loads as pickle_loads 
from marshal import loads as marshal_loads 
from struct import unpack as struct_unpack 
from struct import calcsize 

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 

class unpacker: 
    def __init__(self): 
    self.f = [] 
    self.fcompiled = lambda:None 
    self.sizeofsize = calcsize('Q') 
    def unpack(self, sock): 
    size = struct_unpack('Q', sock.recv(self.sizeofsize))[0] 
    self.f = pickle_loads(sock.recv(size)) 
    a = marshal_loads(self.f[0]) 
    b = globals() ## 
    c = pickle_loads(self.f[1]) 
    d = pickle_loads(self.f[2]) 
    e = pickle_loads(self.f[3]) 
    self.fcompiled = FunctionType(a, b, c, d, e) 
    return self.fcompiled 

test.py

from unpack import unpacker 
from pack import packer 
from sock import SocketServer 
from sock import Socket 
from threading import Thread 
from time import sleep 

count = 2 
port = 4446 

def f(): 
    print 42 

def server(): 
    ss = SocketServer(port) 
    pack = packer() 
    functions = [pack.pack(f) for nothing in range(count)] 
    ss.listen(functions) 

if __name__ == "__main__": 
    Thread(target=server).start() 
    sleep(1) 
    unpack = unpacker() 
    for nothing in range(count): 
    print unpack.unpack(Socket("127.0.0.1", port)) 

输出：

<function f at 0x12917d0> 
<function f at 0x12915f0> 

Answer 1

ValueError: insecure string pickle当你的腌菜被损坏时引发。你确定你在一个sock.recv()（unpack.py）中接收了整个腌制对象吗？

编辑：避免这种情况，你可以做任何大小（你Socket类必须支持接收至与缓冲区大小参数（即

class Socket: 
    def recv(self, bufsize): 
     return self.sock.recv(bufsize) 

）被调用）：

import struct 

struct.pack('Q', len(pickled_list)) 
# Send it, and then send the pickled list. 

在接收机的程序：

import struct 

length = struct.unpack('Q', sock.recv(struct.calcsize('Q')))[0] 
pickled_list = sock.recv(length) 

'Q' 是一种unsigned long long。对于其他结构的东西，请参阅the struct module documentation

Answer 2

我不认为处理对象旨在通过网络发送。查看multiprocessing/process.py中的第256行。

# We subclass bytes to avoid accidental transmission of auth keys over network. 

听起来好像有一个很好的理由给我。如果你想分布式计算，也许你应该看看library designed for that。