See vars.tf for all the variables you can set on this module.
Shard Sizing
Kinesis streams acheive scalability by using shards. This module allows you to either
specify number_of_shards directly or to specify the average_data_size_in_kb, records_per_second and
number_of_consumers variables and the module will calculate the proper number of shards that should be used
based on AWS best practices.
Kinesis streams support server-side encryption as described in the
Kinesis SSE documentation. It can be switched
on retrospectively for existing streams with no interruptions (although only new data will be encrypted).
To enable encryption, set the following parameter
encryption_type = "SSE"
This will use the default AWS service key for Kinesis, aws/kinesis.
{"treedata":{"name":"root","toggled":true,"children":[{"name":".circleci","children":[{"name":"config.yml","path":".circleci/config.yml","sha":"385297a43691f97b5e05ef6c5488955ba0aa577f"}]},{"name":".gitignore","path":".gitignore","sha":"bfcf0f01c753a2c1fa62631296e5678ddd416083"},{"name":"CODEOWNERS","path":"CODEOWNERS","sha":"dd42c2e243782a6f28f5ebafdc16086a8e45a085"},{"name":"LICENSE.txt","path":"LICENSE.txt","sha":"947b78a929481a192a688dbcd8aef5e959102397"},{"name":"README.md","path":"README.md","sha":"1035c5544b541a64aed7073b3998e4cb7b916ab2"},{"name":"examples","children":[{"name":"kinesis","children":[{"name":"README.md","path":"examples/kinesis/README.md","sha":"43778d84a16b259bb5f8b24b299e32776f7d7e98"},{"name":"main.tf","path":"examples/kinesis/main.tf","sha":"3c72361d620e6a4018b9a0c578ffc4f71661821d"},{"name":"outputs.tf","path":"examples/kinesis/outputs.tf","sha":"952a5cf88e478992ff28c5774e8f37217f89d296"},{"name":"vars.tf","path":"examples/kinesis/vars.tf","sha":"9930c9e09cadcc61b822637e0e7f8ea07825596f"}]},{"name":"sns","children":[{"name":"README.md","path":"examples/sns/README.md","sha":"323e05c6b41a125783d38eb76943b26d853164cd"},{"name":"main.tf","path":"examples/sns/main.tf","sha":"ec59117658a70ff2f9cc503f681db986ab6e7480"},{"name":"outputs.tf","path":"examples/sns/outputs.tf","sha":"4f3fdd7d70805e7c7520730b598dca452785ecc5"},{"name":"vars.tf","path":"examples/sns/vars.tf","sha":"b5131e8f6f373e4e08aa5bfb3df0126c668b3db9"}]},{"name":"sqs","children":[{"name":"dead-letter-queue","children":[{"name":"README.md","path":"examples/sqs/dead-letter-queue/README.md","sha":"fbd73ba28e7244fbbed0a460b1659f7535f9c7a0"},{"name":"main.tf","path":"examples/sqs/dead-letter-queue/main.tf","sha":"6090d3241a7bda94938688ae9aa881bbd06b0f9b"},{"name":"outputs.tf","path":"examples/sqs/dead-letter-queue/outputs.tf","sha":"b3d13e984797fab06aeae8abfd599b2a0f12d9f8"},{"name":"vars.tf","path":"examples/sqs/dead-letter-queue/vars.tf","sha":"50cdf37779f5ae015de7e8258f1cbfa0b4b545bb"}]},{"name":"fifo-queue-with-encryption","children":[{"name":"README.md","path":"examples/sqs/fifo-queue-with-encryption/README.md","sha":"3dd276e81e8a297512286032cb59b3c7791f0c09"},{"name":"main.tf","path":"examples/sqs/fifo-queue-with-encryption/main.tf","sha":"7ce2394f0cf8627a6f18198404aeb0300bea0ae0"},{"name":"outputs.tf","path":"examples/sqs/fifo-queue-with-encryption/outputs.tf","sha":"b3d13e984797fab06aeae8abfd599b2a0f12d9f8"},{"name":"vars.tf","path":"examples/sqs/fifo-queue-with-encryption/vars.tf","sha":"7155c480e6a99e1d998ec9a222e2d16378d90ce3"}]},{"name":"no-dead-letter-queue","children":[{"name":"README.md","path":"examples/sqs/no-dead-letter-queue/README.md","sha":"230fa6ad1262430afe23c3e25f6ad5a2dc026281"},{"name":"main.tf","path":"examples/sqs/no-dead-letter-queue/main.tf","sha":"48564f4e525d13253901fd525137323b381e5fbb"},{"name":"outputs.tf","path":"examples/sqs/no-dead-letter-queue/outputs.tf","sha":"e2961511ac23997e4f3e2cd8991559ffdbe10ac0"},{"name":"vars.tf","path":"examples/sqs/no-dead-letter-queue/vars.tf","sha":"a59eb9bc6c66d8a4bf9d9e4d418e78c21035d67b"}]}]}]},{"name":"modules","children":[{"name":"kinesis","children":[{"name":"README.md","path":"modules/kinesis/README.md","sha":"f6976e871de510215552ef64db08d7d878c8a862","toggled":true},{"name":"main.tf","path":"modules/kinesis/main.tf","sha":"67f181f00449b1204c5ff5228d37b46924111695"},{"name":"outputs.tf","path":"modules/kinesis/outputs.tf","sha":"c491c37209211ac71aafe29e4a70e8ddf24a9fc6"},{"name":"vars.tf","path":"modules/kinesis/vars.tf","sha":"11d3a6e29033445cece7839d780cd0b315eb3b16"}],"toggled":true},{"name":"sns","children":[{"name":"README.md","path":"modules/sns/README.md","sha":"5f9aed0e1f0ee934dbbad4907e58bc1e9b5ff1f6"},{"name":"main.tf","path":"modules/sns/main.tf","sha":"569ca2c424c8c8e2ca4d8b14f3cfefbeb7bd9d59"},{"name":"outputs.tf","path":"modules/sns/outputs.tf","sha":"201277ce9ccea18a005fa8e50a205ba505960a34"},{"name":"vars.tf","path":"modules/sns/vars.tf","sha":"aca77a5eb443cd2749332e261be48cb9a87e4066"}]},{"name":"sqs","children":[{"name":"README.md","path":"modules/sqs/README.md","sha":"8efa2221f8f1ab43f5e6021be5da30c66f569b1d"},{"name":"main.tf","path":"modules/sqs/main.tf","sha":"3dc6c6e7007a9eb57599dda1f8b0c860c2607660"},{"name":"outputs.tf","path":"modules/sqs/outputs.tf","sha":"ff29fd711cd239cfaa2ab53f0a9b69124d8f2676"},{"name":"vars.tf","path":"modules/sqs/vars.tf","sha":"576b5dc013f92aba4d34ce18b2dcc084e4ae4c44"}]}],"toggled":true},{"name":"test","children":[{"name":"Gopkg.lock","path":"test/Gopkg.lock","sha":"dc48fe5cdaf54596796722dfaeb8d96713e6502d"},{"name":"Gopkg.toml","path":"test/Gopkg.toml","sha":"3b6f41a0499a5038cb646038da5013a2559976de"},{"name":"README.md","path":"test/README.md","sha":"6985cc4b296c512d47c1ccb205ba11484e6ef1c1"},{"name":"common.go","path":"test/common.go","sha":"0138ee19ba442a095ba70349868b51098830c0b9"},{"name":"kinesis_test.go","path":"test/kinesis_test.go","sha":"58ef5dd9285549020b2c4c92f6d5411588912ca1"},{"name":"sns_test.go","path":"test/sns_test.go","sha":"3536d0fd7acceaa55e30aae64643e77640e57529"},{"name":"sqs_dead_letter_queue_test.go","path":"test/sqs_dead_letter_queue_test.go","sha":"11f10545aecb5485b532a556b98659955afbb83b"},{"name":"sqs_encrypted_fifo_queue_test.go","path":"test/sqs_encrypted_fifo_queue_test.go","sha":"541008f565f1a4317f0f097e19ab5a9a5e702d07"},{"name":"sqs_standard_test.go","path":"test/sqs_standard_test.go","sha":"026c4f86f0f46bc73bf95ca0339322186902124a"}]}]},"detailsContent":"<h1 class=\"preview__body--title\" id=\"kinesis-stream-module\">Kinesis Stream Module</h1><div class=\"preview__body--border\"></div><p>This module makes it easy to deploy a Kinesis stream</p>\n<h2 class=\"preview__body--subtitle\" id=\"how-do-you-use-this-module\">How do you use this module?</h2>\n<ul>\n<li>See the <a href=\"/repos/v0.4.1/package-messaging/README.md\" class=\"preview__body--description--blue\">root README</a> for instructions on using Terraform modules.</li>\n<li>See the <a href=\"/repos/v0.4.1/package-messaging/examples\" class=\"preview__body--description--blue\">examples</a> folder for example usage.</li>\n<li>See <a href=\"/repos/v0.4.1/package-messaging/modules/kinesis/vars.tf\" class=\"preview__body--description--blue\">vars.tf</a> for all the variables you can set on this module.</li>\n</ul>\n<h2 class=\"preview__body--subtitle\" id=\"shard-sizing\">Shard Sizing</h2>\n<p>Kinesis streams acheive scalability by using <a href=\"https://en.wikipedia.org/wiki/Shard_(database_architecture)\" class=\"preview__body--description--blue\" target=\"_blank\">shards</a>. This module allows you to either\nspecify <code>number_of_shards</code> directly or to specify the <code>average_data_size_in_kb</code>, <code>records_per_second</code> and\n<code>number_of_consumers</code> variables and the module will calculate the proper number of shards that should be used\nbased on <a href=\"https://docs.aws.amazon.com/streams/latest/dev/amazon-kinesis-streams.html\" class=\"preview__body--description--blue\" target=\"_blank\">AWS best practices</a>.</p>\n<p><code>incoming_write_bandwidth_in_kb = average_data_size_in_kb * records_per_second</code></p>\n<p><code>outgoing_read_bandwidth_in_kb = incoming_write_bandwidth_in_kb * number_of_consumers</code></p>\n<p><code>number_of_shards = max(incoming_write_bandwidth_in_kb/1000, outgoing_read_bandwidth_in_kb/2000)</code></p>\n<h2 class=\"preview__body--subtitle\" id=\"encryption\">Encryption</h2>\n<p>Kinesis streams support server-side encryption as described in the\n<a href=\"https://docs.aws.amazon.com/streams/latest/dev/what-is-sse.html\" class=\"preview__body--description--blue\" target=\"_blank\">Kinesis SSE documentation</a>. It can be switched\non retrospectively for existing streams with no interruptions (although only new data will be encrypted).</p>\n<p>To enable encryption, set the following parameter</p>\n<p><code>encryption_type = "SSE"</code></p>\n<p>This will use the default AWS service key for Kinesis, <code>aws/kinesis</code>.</p>\n<p>If you need to use a custom key, see the\n<a href=\"/repos/module-security-public/modules/kms-master-key\" class=\"preview__body--description--blue\">master key module</a> as well as\n<a href=\"https://docs.aws.amazon.com/streams/latest/dev/creating-using-sse-master-keys.html\" class=\"preview__body--description--blue\" target=\"_blank\">documentation on user-generated KMS master keys</a>\nfor further information on how to create them. You can specify one using</p>\n<p><code>kms_key_id = "alias/<my_cmk_alias>"</code></p>\n<h2 class=\"preview__body--subtitle\" id=\"examples\">Examples</h2>\n<p>Here are some examples of how you might deploy a Kinesis stream with this module:</p>\n<pre>module <span class=\"hljs-string\">\"kinesis\"</span> {\n <span class=\"hljs-attr\">source</span> = <span class=\"hljs-string\">\"git::git@github.com:gruntwork-io/package-messaging.git//modules/kinesis?ref=v0.0.1\"</span>\n\n <span class=\"hljs-attr\">name</span> = <span class=\"hljs-string\">\"my-stream\"</span>\n <span class=\"hljs-attr\">retention_period</span> = <span class=\"hljs-number\">48</span>\n\n <span class=\"hljs-attr\">number_of_shards</span> = <span class=\"hljs-number\">1</span>\n <span class=\"hljs-attr\">shard_level_metrics</span> = [\n <span class=\"hljs-string\">\"IncomingBytes\"</span>,\n <span class=\"hljs-string\">\"IncomingRecords\"</span>,\n <span class=\"hljs-string\">\"IteratorAgeMilliseconds\"</span>,\n <span class=\"hljs-string\">\"OutgoingBytes\"</span>,\n <span class=\"hljs-string\">\"OutgoingRecords\"</span>,\n <span class=\"hljs-string\">\"ReadProvisionedThroughputExceeded\"</span>,\n <span class=\"hljs-string\">\"WriteProvisionedThroughputExceeded\"</span>\n ]\n \n}\n</pre>\n<pre>module <span class=\"hljs-string\">\"kinesis\"</span> {\n <span class=\"hljs-attr\">source</span> = <span class=\"hljs-string\">\"git::git@github.com:gruntwork-io/package-messaging.git//modules/kinesis?ref=v0.0.1\"</span>\n <span class=\"hljs-attr\">name</span> = <span class=\"hljs-string\">\"my-stream\"</span>\n <span class=\"hljs-attr\">retention_period</span> = <span class=\"hljs-number\">48</span>\n \n <span class=\"hljs-attr\">average_data_size_in_kb</span> = <span class=\"hljs-number\">20</span>\n <span class=\"hljs-attr\">records_per_second</span> = <span class=\"hljs-number\">10</span>\n <span class=\"hljs-attr\">number_of_consumers</span> = <span class=\"hljs-number\">10</span>\n \n <span class=\"hljs-attr\">shard_level_metrics</span> = [\n <span class=\"hljs-string\">\"ReadProvisionedThroughputExceeded\"</span>,\n <span class=\"hljs-string\">\"WriteProvisionedThroughputExceeded\"</span>\n ]\n}\n</pre>\n","repoName":"package-messaging","repoRef":"v0.3.1","serviceDescriptor":{"serviceName":"Kinesis","serviceRepoName":"package-messaging","serviceRepoOrg":"gruntwork-io","serviceMainReadmePath":"/modules/kinesis","cloudProviders":["aws"],"description":"Create Kinesis streams with configurable or auto-calculated shard and retention settings.","imageUrl":"kinesis.png","licenseType":"subscriber","technologies":["Terraform"],"compliance":[],"tags":[""]},"serviceCategoryName":"Messaging & streaming","fileName":"README.md","filePath":"/modules/kinesis","title":"Repo Browser: Kinesis","description":"Browse the repos in the Gruntwork Infrastructure as Code Library."}