使用python将所选文件从sftp加载到S3

我在sftp上有一些文件（名为a.csv，b.csv，c.csv），我想使用python直接将该文件加载到S3。我从Stackoverflow得到下面的代码，但问题是，而不是ftp_priv_key_filename，我使用密码连接到sftp。请给我建议正确的做法。由于使用python将所选文件从sftp加载到S3

import math 
import os 

from boto.s3.connection import S3Connection 
import io 
import paramiko 
import stat 
import time 


chunk_size = 12428800 
ftp_priv_key_filename = '/path/to/private/key' # this script assume priv use auth 
ftp_username = 'user' 
ftp_host = '100.10.86.59' 
ftp_port = 22 
ftp_dir = '/import/TMP' 
s3_id = 'abc' 
s3_key = 'xyz' 
bucket_name = 'efg/mno/pqr' 


s3_conn = S3Connection(s3_id, s3_key) 
bucket = s3_conn.get_bucket(bucket_name) 


pkey = paramiko.RSAKey.from_private_key_file(ftp_priv_key_filename) 
transport = paramiko.Transport((ftp_host, ftp_port)) 
transport.connect(username=ftp_username, pkey=pkey) 
ftp_conn = paramiko.SFTPClient.from_transport(transport) 


def move_file(filepath): 
    key_id = filepath.replace(ftp_dir, '').lstrip('/') 
    key = bucket.get_key(key_id) 
    ftp_fi = ftp_conn.file(filepath, 'r') 
    source_size = ftp_fi._get_size() 
    if key is not None: 
     # check if we need to replace, check sizes 
     if source_size == key.size: 
      print('%s already uploaded' % key_id) 
      ftp_fi.close() 
      return 

    chunk_count = int(math.ceil(source_size/float(chunk_size))) 
    mp = bucket.initiate_multipart_upload(key_id) 

    print('%s uploading size: %imb, %i chunks' % (
     key_id, math.ceil(source_size/1024/1024), chunk_count)) 
    for i in range(chunk_count): 
     start = time.time() 
     chunk = ftp_fi.read(chunk_size) 
     end = time.time() 
     seconds = end - start 
     print('%s read chunk from ftp (%i/%i) %ikbs' % (
      key_id, i + 1, chunk_count, 
      math.ceil((chunk_size/1024)/seconds))) 

     fp = io.BytesIO(chunk) 
     start = time.time() 
     mp.upload_part_from_file(fp, part_num=i + 1) 
     end = time.time() 
     seconds = end - start 
     print('%s upload chunk to s3 (%i/%i) %ikbs' % (
      key_id, i + 1, chunk_count, 
      math.ceil((chunk_size/1024)/seconds))) 

    mp.complete_upload() 
    ftp_fi.close() 


def move_dir(directory): 
    ftp_conn.chdir(directory) 
    for filename in ftp_conn.listdir(): 
     filepath = os.path.join(directory, filename) 
     if stat.S_ISDIR(ftp_conn.stat(filepath).st_mode): 
      move_dir(filepath) 
     else: 
      move_file(filepath) 

move_dir(ftp_dir)

来源

2017-02-27 James

你只需要拿出下面一行：

pkey = paramiko.RSAKey.from_private_key_file(ftp_priv_key_filename)

和修改FTP连接到以下（基本上使用密码，而不是私钥）

transport.connect(username=ftp_usernamem=, password=ftp_password)

来源

2017-06-14 22:05:41 Mariah

使用python将所选文件从sftp加载到S3

回答

相关问题