Pi4J + Amazon S3 REST API + S3 Lambda Trigger + DynamoDB [P022]

Pi4J + Amazon S3 REST API + S3 Lambda Trigger + DynamoDB [P022]

https://www.youtube.com/watch?v=LHZBQmybbqc

GitHub : https://github.com/rdiot/rdiot-p022

* Parts

- Pi4J + Amazon S3 REST API + Amazon Athena [P019]

- Raspberry Pi2

- Temperature and humidity DHT11 sensor module (KY-015) [S008]

* Contents

1. S3 Upload 

- Reference : Pi4J + Amazon S3 REST API + Amazon Athena [P019]

2. Run Java Application (pi4j)

- Source : 

$ java -Dpi4j.linking=dynamic -jar pi4j_s3rest-0.0.1-SNAPSHOT.jar

3.  S3 Storage Monitoring

   

4. AWS Lambda Monitoring

5. AWS DynamoDB Monitoring

6. AWS Lambda Source (node.js) : S3toDynamoDB

- source code : https://github.com/rdiot/rdiot-p022/blob/master/S3toDynamoDB/index.js

'use strict';

console.log('RDIoT S3toDynamoDB Loading post function');

var AWS = require('aws-sdk'); 

var s3 = new AWS.S3();

var dynamo = new AWS.DynamoDB.DocumentClient();

exports.handler = function(event, context, callback) {

    console.log('Received event:', JSON.stringify(event, null, 2));

    var bucket = event.Records[0]['s3']['bucket']['name'];

    var en = event.Records[0]['eventName'];

    var et = event.Records[0]['eventTime'];

    var key = event.Records[0]['s3']['object']['key'];

    var sip = event.Records[0]['requestParameters']['sourceIPAddress'];

    var params1 = {Bucket: bucket, Key: key}; 

    s3.getObject( params1, function(err,data) {

        if(err) {

            console.log(err.stack);

            callback(err);

        } else {

            var value = data.Body.toString('ascii')

            console.log(data);

            console.log("Raw text:\n" + value);

            var params2 = {

                TableName: "pi-sensor",

                Item:{

                    "id": et,

                    "event" : en,

                    "bucket": bucket,

                    "key" : key,

                    "sip": sip,

                    "value" : value

                }

            };

            console.log("Gettings IoT device details...");

            //S3 to DynamoDB

            dynamo.put(params2, function(err, data) {

                if(err) {

                    console.error("Unable to post devices. Error JSON:", JSON.stringify(err, null, 2));

                    context.fail();

                } else {

                    console.log("keepet data:", JSON.stringify(data, null, 2));

                    context.succeed('success post');

                }

            });

        }

    });

}

7. Lambda : s3 put event sample

{

    "Records": [

      {

        "awsRegion": "ap-northeast-2",

        "eventName": "ObjectCreated:Put",

        "eventSource": "aws:s3",

        "eventTime": "2018-06-30T12:04:47.253Z",

        "eventVersion": "2.0",

        "requestParameters": {

          "sourceIPAddress": "211.245.211.252"

        },

        "responseElements": {

          "x-amz-id-2": "PTHGAl4fTlMHW/2lurF0HaruzgLtTnG1JyUlWC9o9AM21jLLUQ5Dp3LH1Ur0uVltB6L7BqDs6Ts=",

          "x-amz-request-id": "31CCB31CF0807321"

        },

        "s3": {

          "bucket": {

            "arn": "arn:aws:s3:::rdiot-test",

            "name": "rdiot-test",

            "ownerIdentity": {

              "principalId": "A3PLNTCS5VC05F"

            }

          },

          "configurationId": "174810a9-e50b-484e-bfda-eafed1ff2ec3",

          "object": {

            "eTag": "5d81121100d6ec386ee75237c8eb3549",

            "key": "pi_dht11_20186300444.json",

            "sequencer": "005B3771DF3531F6C1",

            "size": 35

          },

          "s3SchemaVersion": "1.0"

        },

        "userIdentity": {

          "principalId": "AWS:AIDAIQKJB3IKQS4YDOI2Y"

        }

      }

    ]

  }

8. Lambda : Context Sample

{

    "awsRequestId": "45bca188-7c5e-11e8-9d80-cdcb57e017f4",

    "callbackWaitsForEmptyEventLoop": true,

    "functionName": "S3toDynamoDB",

    "functionVersion": "$LATEST",

    "invokedFunctionArn": "arn:aws:lambda:ap-northeast-2:996409770277:function:S3toDynamoDB",

    "invokeid": "45bca188-7c5e-11e8-9d80-cdcb57e017f4",

    "logGroupName": "/aws/lambda/S3toDynamoDB",

    "logStreamName": "2018/06/30/[$LATEST]16b94689af7d4257bcc75dffc45798db",

    "memoryLimitInMB": "128"

  }

9. Lambda : S3 Object GET Data Sample

2018-06-30T14:04:12.852Z 775a8d39-7c6e-11e8-bc88-e9ab4a76e0af { AcceptRanges: 'bytes',

  LastModified: 2018-06-30T12:04:48.000Z,

  ContentLength: 35,

  ETag: '"5d81121100d6ec386ee75237c8eb3549"',

  CacheControl: 'no-cache',

  ContentType: 'binary/octet-stream',

  Metadata: {},

  StorageClass: 'REDUCED_REDUNDANCY',

  Body: <Buffer 7b 22 6e 61 6d 65 22 3a 22 74 65 6d 70 65 72 61 74 75 72 65 22 2c 22 76 61 6c 75 65 22 3a 33 30 2e 30 7d> }

2018-06-30T14:04:12.853Z 775a8d39-7c6e-11e8-bc88-e9ab4a76e0af Raw text:

{"name":"temperature","value":30.0}