Binary Parser

Can be configured to parse a wide variety of binary formats. See Supported reader-parser combinations for compatible readers.

Binary Parser properties

property	type	default value	notes
Endian	Boolean	`False`	`False` means the format is little endian, set to `True` if the format is big endian
Metadata	String		path from root or relative to the Striim program directory and the name of JSON file defining how to parse the data
String Terminated By Null	Boolean	`True`	With the default value of `True`, strings must be terminated by a single null character (ASCII 00). If set to `False`, string length must be defined in the JSON file specified by the `metadata` property.

The output type of a source using BinaryParser is WAEvent.

Binary Parser data types

The data types supported by the metadata JSON are:

type	length in bytes
BYTE	1
DOUBLE	8
FLOAT	4
INTEGER	4
LONG	8
SHORT	2
STRING	as set by stringColumnLength

Binary Parser examples

For a sample application, download BinaryParser.zip, which contains insorders.c, metadata.json, and bin.tql, from https://github.com/striim/doc-downloads. To run this sample application:

Run insorders.c and copy its test.bin output to the Samples directory.

/***** insorders.c ******************************************
**                                                          *
** Generate order records in binary format                  *
** Usage insorders <recordcount>                            *
** Max recordcount 1000000                                  *
** Note                                                     *
**                                                          *
************************************************************/
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
/* Define some useful constants */
#define HIGH  1
#define MEDIUM 2
#define LOW 3
#define PRICE_TYPES 3
#define FILENAME2               "/Users/mahadevan/binary/test.bin"
#define MAX_LEN_CNAME           20
#define MAX_DIG_CNAME           10
#define NAMELEN                 20
#define MAX_ITEM_COUNT          200
#define EXPENSIVE_ITEM_PRICE    745.99
#define MEDIUM_ITEM_PRICE       215.99
#define LOW_ITEM_PRICE          34.79
 
/* Function prototypes */
int testdata(int numRecords);
double setPrice(int cid);
int main(int argc, char *argv[])
{
  int count = 1000;
  int ret = 1;
  /* just basic error checking for now is sufficient */
  
  
   printf(" Inserting %d Records\n", count);
  
  
  if ((ret = testdata(count)) !=0)
     printf("TestData Failed\n");
} /* End Main */
/* Start Function for test data generation */
int testdata(int numRecords)
{
  int i,nm1,nm2 = 0;
  
  /* Declare variables for random record creation \n")*/  
  int order_id=10000;
  int district_id=0;
  int warehouse_id = 100;
  char cname[numRecords][MAX_LEN_CNAME];  
  char PrefixName[NAMELEN]="John";
  char numRecStr[MAX_DIG_CNAME];
  int count_id=0;
  double price=0;
  /* time_t now; Just generate sysdate for now */
  bzero(numRecStr,MAX_DIG_CNAME);
  FILE *fptr_bin =fopen(FILENAME2, "w");
  if (fptr_bin==NULL) return -1;
   for (nm1=0; nm1 < numRecords; nm1++)
   {
      sprintf(numRecStr, " %d",nm1); 
      strcat(PrefixName,numRecStr); 
      strcpy(cname[nm1],PrefixName); 
      //printf("Generated Name is %s\n", cname[nm1]);
      /* Re-init to base root for name */
      strcpy(PrefixName, "John");
      /* Generate a random count of items between 0 and 20 */
      count_id = rand()%MAX_ITEM_COUNT;
      price = setPrice(count_id);
      
      printf("Price is %f\n",price);
      short cnamelen = strlen(cname[nm1]);
      /* Generate record with the following fields */
      fwrite((const void *) (&order_id), sizeof(int), 1, fptr_bin); order_id++;
      fwrite((const void *) (&district_id), sizeof(int), 1, fptr_bin); district_id++;
      fwrite((const void *) (&warehouse_id), sizeof(int), 1, fptr_bin); warehouse_id++;
      fwrite((const void *) (&cnamelen), sizeof(short), 1, fptr_bin); 
      fwrite((const void *) (cname[nm1]), sizeof(char), strlen(cname[nm1]), fptr_bin); 
      fwrite((const void *) (&count_id), sizeof(int), 1, fptr_bin); count_id++;
      fwrite((const void *) (&price), sizeof(double), 1, fptr_bin); 
  }
  fclose(fptr_bin);
  return 0;
}
/* Start setPrice */
  double setPrice(int cid)
  {
    short i;
    double price, total_price=0;
    
    i = rand()%PRICE_TYPES;
    switch (i) 
    {
      case(HIGH):
          price = EXPENSIVE_ITEM_PRICE;
      case(MEDIUM):
          price = MEDIUM_ITEM_PRICE;
      case(LOW):
          price = LOW_ITEM_PRICE;
     }
     total_price = cid*price;
     return total_price;
  } /* End setPrice */

Copy metadata.json to the Samples directory.

{
     "namespace": "test",
     "name": "SensorFeed",
     "version": "1.0",
     "type": "record",
     "fields": [
       {
        "name": "order_id",
        "type": "INTEGER",
         "position":"0"
        },        
       {
        "name": "district_id",
        "type": "INTEGER",
         "position":"1"
        },        
       {
        "name": "warehouse_id",
        "type": "INTEGER",
         "position":"2"
        },        
       {
        "name": "customer_name",
        "type": "STRING",
         "position":"3"
        },        
       {
        "name": "count_id",
        "type": "INTEGER",
         "position":"4"
        },        
       {
        "name": "price",
        "type": "DOUBLE",
         "position":"5"
        }        
      ]
    }

Load bin.tql and run the bin application.

CREATE APPLICATION bin;
 
CREATE SOURCE BinarySource using FileReader (
  directory:'Samples',
  wildcard:'test.bin',
  positionByEOF:false
)
PARSE USING BinaryParser (
  metadata:'Samples/metadata.json',	
  endian:true
)
OUTPUT TO BinaryStream;

CREATE TARGET BinaryDump USING LogWriter(
  name:BinaryReader, 
  filename:'out.log'
) INPUT FROM BinaryStream;

CREATE TYPE OrderType (
  OrderId integer,
  DistrictId integer,
  WarehouseId integer,
  Name String,
  CountId integer,
  Price double);
     
CREATE STREAM OrderStream OF OrderType;

CREATE CQ OrderCQ 
INSERT into OrderStream
SELECT data[0],
  data[1],
  data[2],
  data[3],
  data[4],
  TO_DOUBLE(data[5])
FROM BinaryStream;

CREATE TARGET DSVOut
USING FileWriter(filename: binary_output)
FORMAT USING DSVFormatter()
INPUT FROM OrderStream;

END APPLICATION bin;

The binary_output.csv output file will start like this:

10000,0,100,John 0,7,243.53
10001,1,101,John 1,73,2539.67
10002,2,102,John 2,130,4522.7
10003,3,103,John 3,144,5009.76
10004,4,104,John 4,123,4279.17
10005,5,105,John 5,40,1391.6

Binary Parser

Binary Parser properties

Binary Parser data types

Binary Parser examples

Search results