如何在dynamodb中超过1mb的扫描数据限制

Question

我正在使用dynamodb和nodejs，我有3000条记录，并且我在代码中编写了60多个段，每段扫描1mb数据并显示结果到60多个1MB的限制。因此，请提供解决方案，如何在单个步骤中获得3000条记录扫描，这意味着在一个分段中。请迅速提供解决方案，因为我在项目中间受到了攻击。请帮帮我。以下是我的代码。

var AWS = require("aws-sdk"); 
var async = require("async"); 

    AWS.config.update({ 
     region: "---", 
     endpoint: "-----------", 
     accessKeyId: "-----------------", 
     secretAccessKey:"----------" 
    }); 


var db = new AWS.DynamoDB.DocumentClient() 

var table = "rets_property_all"; 
var pstart =new Date() .getTime(); 

async.parallel({ 
      0 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 0//by the worker who has been called 
     },function (err , res) { 
      callback (null , res.Items); 
     }); 
      }, 
      1 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 1//by the worker who has been called 
     }, function (err , res) { 
      callback (null , res.Items); 
     }); 
      }, 
     2 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 2//by the worker who has been called 
     }, function (err , res) { 
      callback (null , res.Items); 
     }); 
      }, 
-------- 
--------- 
------ 

     62 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 62//by the worker who has been called 
     }, function (err , res) { 
      callback (null , res.Items); 
     }); 
     }, 

     },function(err,results){ 
     if (err) {throw err; } 
     var pend = new Date() .getTime(); 

     console.log (results); 
     }) 

Answer 1

其实，有没有办法覆盖扫描1 MB限制。这是DynamoDB设计限制，不能被任何API覆盖。您可能需要了解体系结构或AWS服务设计的限制。

您可以在随后的扫描中使用LastEvaluatedKey从上次扫描结束时开始。

扫描的结果集限于每次调用1 MB。您可以使用扫描响应中的LastEvaluatedKey 来检索更多结果。

用例不清楚为什么你想在一次扫描中获得所有3000条记录。即使您有特定用例，也无法在DynamoDB扫描中实现。

即使在关系型数据库中，您也可以获取游标并遍历迭代以顺序获取所有记录。同样，在DynamoDB中，您必须递归使用Scan，直到LastEvaluatedKey为空。单扫描

示例代码递归直到LastEvaluatedKey为空： -

var docClient = new AWS.DynamoDB.DocumentClient(); 
var params = { 
    TableName: table, 
    ProjectionExpression: "#cityname,ListingKey ", 
    FilterExpression: "#cityname = :v_id", 
    ExpressionAttributeNames: { 
     "#cityname": "CityName", 
    }, 
    ExpressionAttributeValues: { ":v_id": 'BALTIMORE' } 
}; 

docClient.scan(params, onScan); 

function onScan(err, data) { 
    if (err) { 
     console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2)); 
    } else { 
     // print all the movies 
     console.log("Scan succeeded."); 
     data.Items.forEach(function (movie) { 
      console.log("Item :", JSON.stringify(movie)); 
     }); 

     // continue scanning if we have more movies 
     if (typeof data.LastEvaluatedKey != "undefined") { 
      console.log("Scanning for more..."); 
      params.ExclusiveStartKey = data.LastEvaluatedKey; 
      docClient.scan(params, onScan); 
     } 
    } 
}