Local data storage

All the data received by the collector, independently of the format, is either stored locally and sent to Intelie Live as-is. That is, no modification is performed on the collected data inside the collector.

All the data is stored in the file /opt/intelie/liverig/data/data.log. Simultaneously, a thread pool is reading this file and sending all of the data to Intelie Live. The process takes just a few milliseconds.

In this file, it is possible to see the original data (3), and also some control information (1) and some metadata (2) that are created and transmitted together, as described below.

1. control information = marked by info line, a pure text signalling if the payload next is an internal control event (=true) or the protocol data itself (=false)

2. some metadata = marked by meta line, also in pure text containing all metadata collected from the original application protocol

3. original data = marked by data line, the payload after the first space is the protocol payload length and its verbatim format which can be text or binary or mixed

Demonstration of a snippet of WITSML data gathering

93:0 info internal=false
93:0 meta 568 {"program_name":"liverig-collector","program_version":"5.26.12-SNAPSHOT","instance":"live","source_name":"Pason","rig_name":"pason_v1311","source_kind":"Pason","source_mode":"client","source_protocol_name":"witsml","source_protocol_version":"1.3.1.1","capture_start_timestamp":"1761160141852","capture_timestamp":"1761160147087","requested_start_date":"2025-10-22T16:39:01.847Z","log_delay":"0","timestamp_adjustment":"0","timezone_offset":-10800000,"data_crc32":"720944053","ignoreData":{"lastChangeValue":1.761151141847E12,"lastChangeValueUnit":""},"revision":false}
93:0 data 695637 <?xml version='1.0' encoding='UTF-8'?>
<logs xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.witsml.org/schemas/131" xsi:schemaLocation="http://www.witsml.org/schemas/131 ../obj_log.xsd" version="1.3.1.1"><log uidWell="de_1759454946" uidWellbore="de_1759454946_wb1" uid="de_1759454946_wb1_log_hfll_time_1s"><nameWell>FAKE-FIELD 24A</nameWell><nameWellbore>FAKE-FIELD 24A WB #1</nameWellbore><name>Log #hfll_time_1s</name><objectGrowing>1</objectGrowing><dataRowCount>1749180</dataRowCount><indexType>date time</indexType><stepIncrement uom="ft">0</stepIncrement><startDateTimeIndex>2025-10-22T11:39:01-05:00</startDateTimeIndex><endDateTimeIndex>2025-10-22T11:41:35-05:00</endDateTimeIndex><direction>increasing</direction><indexCurve columnIndex="1">TIME</indexCurve><logCurveInfo uid="lci-time"><mnemonic>TIME</mnemonic><unit>unitless</unit><minDateTimeIndex>2025-10-22T11:39:01-05:00</minDateTimeIndex><maxDateTimeIndex>2025-10-22T11:41:35-05:00</maxDateTimeIndex><columnIndex>1</columnIndex><curveDescription>Time Logged</curveDescription><typeLogData>date time</typeLogData></logCurveInfo><logCurveInfo uid="lci-a1"><mnemonic>A1</mnemonic><unit>UNITS</unit><minDateTimeIndex>2025-10-22T11:39:01-05:00</minDateTimeIndex><maxDateTimeIndex>2025-10-22T11:41:35-05:00</maxDateTimeIndex><columnIndex>2</columnIndex><curveDescription>Algorithm #1 (UNITS)</curveDescription><typeLogData>double</typeLogData></logCurveInfo><logCurveInfo uid="lci-a10"><mnemonic>A10</mnemonic><unit>UNITS</unit><minDateTimeIndex>2025-10-22T11:39:01-05:00</minDateTimeIndex><maxDateTimeIndex>2025-10-22T11:41:35-05:00</maxDateTimeIndex><columnIndex>3</columnIndex><curveDescription>Algorithm #10 (UNITS)</curveDescription><typeLogData>double</typeLogData></logCurveInfo><logCurveInfo uid="lci-a2"><mnemonic>A2</mnemonic><unit>UNITS</unit><minDateTimeIndex>2025-10-22T11:39:01-05:00</minDateTimeIndex><maxDateTimeIndex>2025-10-22T11:41:35-05:00</maxDateTimeIndex><columnIndex>4</columnIndex><curveDescription>
... ommited ...
<data>2025-10-22T11:41:35-05:00,2460.73,2464.19,4666.07,9.70,-52.748,-76.448,0,1776.37,1.622,0.481,0.000,11.39,-395.361,0,0,0.0,22239,0,65.00,2000.0,0.0,0,0,0.00,0.00,0,0,8198,12.424,,,,,15.00,-15.00,875.0,675.0,36.00,-36.00,,,1168,,,,13.86,-1.00,-120.5060,23.0000,105.4440,,440,60,50,,0,165.37,5832,8.13,12576.3,16.9,1,140,88,96,-29997.00,2000.00,0.00,10.000,0.0000,1.00,0.00,8.77,85.54,9.05,32.73,-0.254,6.672,6.55,14.80,5.60,23.80,3.00,798.000,3.70,4.40,20.80,35.70,8.77,-9999.00,-9999.00,-9999.000,,1.00,0.00,0.00,,0.00,0.00,0.00,0.00,6.42,832.77,,,,,-1.00,0.00,,,0.00,0.07,,,,,,0.05,0.09,0.14,0.17,0.01,0.00,0.00,0.14,0.00,0.00,,0.00,1.00,1.00,,,7577.34,7717.33,1076.38,6789.30,20.90,15.32,70.17,0.00,0.00,,0.00,0.00,0.00,0.03,0.04,0.00,0.00,0.00,,,23292.60,0.07,,,,,22.22,173.60,-7.482,0.00,67.814,0.00,1.00,0.000,0.000,0.000,0.000,0.000,0.000,17.676,11.650,40.094,40.058,0.00,0.000,250.000,402.171,0.00,-9999.00,-9999.00,-9999.00,1356.130,157.413,69.464,11.38,15.000,-8.202,400.104,96.6,103.9,,,,1.03,182.9,149.1,11.650,0,,0.00,0.64,,0.0,,,,,,,,,,0.0,0.0,,0.79,6.67,15.57,,876.00,,0.00,120.00,244.35,0,0,3,1,0.00,4.0,0,,,7,4,25,0,0,,,,,39.370,,1250.0,0.0,0.000,0,15308.6,,66.5,50.00,,69.46,,,0.000,0,,0,0,1150.0,1,3,80.0,120.0,2342,650.000,,650.0,,0,,0,0.00,0.000,15.0,35.0,165.52,90.20,-9.98,23.72,0.0,95.0,0.0,512,47.0,5.0,0.0,0,2.0,11,0,0.0,0.0,640,5.0,0.0,0.0,128,11,0.18,25.382,1.622,0,26.7,26.7,512,0.0,5.6,26.74,0.0,25,0.00,15.00,0.01,39.00,0.00,99.86,99.86,,34.0,16384.0,0.34,86.5,12543.18,770.38,770.38,770.38,105.44,,6.00,0.000,8.750,157.4,50,913.0,432.8,90.98,0,45,187205,0,0.000,23292,0.002,0.002,32.73,8.77,6.55,9.05,0.001,23.80,-0.25,6.67,798.0,14.80,8.77,0,1.0,85.5,0.0,3.0,0.003,11.271,8.7,8.4,8.3,640,10.00,156.8,1.0,0.0,2044.006,1.00,18293.013,1336.63,40.03,,,,7.7,31.16,,11272,48,57020700,55342700,49717000,1336.6,1337,770.38,44.0,9.76,9.38,9.76,-8.4,84.951,84.57,1287,0.00,0,2,0.000,120.7,9.00,176.19,0.43,0.00,0.00,3724750.000,12,1.90,0.28,0.13,0.14,0.00,0.00,769.860,0.508,0.0,0,3724750.000,2.45,2.45,0.27,0.06,15308.6,-401.706,10.00,930.0,-349.72,0.07,0.00,0.00,0.0368,0.000,17.676,2.45,6.8,3.2,15375.46,0.00,5.3,169.9276,0.0,6319.75,776.5,763.2,1.0,0.00,35.0,157.0,89.7,0,0.0,1.00,120,35.0,1250.0,80,15.0,3,650.0,,0.00,0.30,18459,40.06,134.0,2.0,0.0,7.00,7.00,0.0,0.0,0.0,0,0.0,0.270,1222.3,54,69.89,33.94,47.8,0,15250.0,54.98,0.00,0.00,1356.13,,37,290,31,250.00,170.00,150.00,260.00,256.00,80.00,105.00,300.00,0,0.500,0.056,5000,0,0,,40.094,11.650,105.44,105,0,0.00,425.0,225.0,40.0,0,-10,-10,0.00,440,652442,-1576360000,3001,1,,,130.5,80,1.633,,,0.00,,2.45,0.34,1152,15.00,640,875.00,-15.00,675.00,0,0,0.00,11649.600,85410200,122811000,100416000,3489335.0,12,176.19,1.0,36.00,-36.00,0.33,6.85,7687,7687,0,0,0,128.0,41.00,8.00,128.0,41.00,8.00,-138.17,48.00,48.00,9.755,0.000,1.26,50.00,1.00,16.045,3.466,187.81,128.24,106.63,174.40,173.29,67.87,91.10,273.86,1356,1356.1,0,-1.91,2,0.00,,22.25,0.000,1.000,1.000,,,,1.000,1.000,,,5488.420,0.00,1400.000,5.00,2.00,5.00,1.00,4.000,0.00,0.000,222.00,,21300.0,0.0,0,-8.2,,0,0.00,0.000,0.000,85.000,0.000,0.0,0.800,2.000,0.000,22,0.000,,30.000,0.000</data></logData><commonData><sourceName>Pason Systems</sourceName><dTimCreation>2025-10-02T20:29:09-05:00</dTimCreation><itemState>actual</itemState></commonData></log></logs>

Demonstration of a internal event

92:0 info internal=true
92:0 meta 253 {"program_name":"liverig-collector","program_version":"5.26.12-SNAPSHOT","instance":"live","source_name":"__internal__","source_kind":"__internal__","capture_start_timestamp":"1761159858879","capture_timestamp":"1761159859075","data_crc32":"2171616120"}
92:0 data 9900 {"freeMemory":"1828025056","totalMemory":"2147483648","maxMemory":"2147483648","storageUsedEvents":"2","storageUsedBytes":"260118099","receivedBytesControl":"188","receivedBytesRealtime":"0","receivedBytesResent":"0","receivedBytesRemoteControl":"0","sentBytesControl":"296","sentBytesRealtime":"0","sentBytesResent":"0","sentBytesRemoteControl":"0","receivedBytesCompressed":"0","receivedBytesUncompressed":"0","sentBytesUncompressed":"0","sentBytesCompressed":"0","protocolErrors":"0","secure":true,"perSourceCounters":{},"config":{"sources":[{"tls_auth":false,"service_company":"Intelie","filters":{},"requests":[],"enabled":false,"mode":"server","name":"pipes (amadeu)","id":1,"rig_name":"amadeu","protocol_name":"raw"},{"tls_auth":false,"service_company":"Intelie","filters":{},"requests":[],"enabled":false,"mode":"server","name":"pipes (barbosa)","id":2,"rig_name":"barbosa","protocol_name":"raw"},{"tls_auth":false,"service_company":"Intelie","filters":{},"requests":[],"enabled":false,"mode":"server","name":"pipes (bla)","id":3,"rig_name":"bla","protocol_name":"raw"},{"tls_auth":false,"service_company":"Intelie","filters":{},"requests":[],"enabled":false,"mode":"server","name":"pipes (bli)","id":4,"rig_name":"bli","protocol_name":"raw"},{"tls_auth":false,"service_company":"Intelie","filters":{},"requests":[],"enabled":false,"mode":"server","name":"pipes (ble)","id":5,"rig_name":"ble","protocol_name":"raw"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"offset":0,"uid_log":"us_30171200_wb1_log_hfll_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30171200","log_delay":0,"id":1,"query_period":10,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h/log/us_30171200_wb1_log_hfll_time_1s","object":"log"},{"revision_waiting_time":0,"offset":0,"uid_log":"us_30171200_wb1_log_dfr_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30171200","log_delay":0,"id":2,"query_period":10,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h/log/us_30171200_wb1_log_dfr_time_1s","object":"log"},{"revision_waiting_time":0,"offset":0,"uid_log":"us_30171200_wb1_log_1","filters":{},"enabled":false,"index_type":"measured depth","uid_well":"us_30171200","log_delay":0,"id":3,"query_period":30,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h/log/us_30171200_wb1_log_1","object":"log"},{"revision_waiting_time":0,"offset":0,"uid_log":"*","filters":{},"enabled":false,"uid_well":"us_30171200","log_delay":0,"id":4,"query_period":600,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h/message","object":"message"}],"enabled":false,"mode":"client","protocol_version":"1.3.1.1","name":"SPOONBILL-DONOVAN 10F 6H","id":6,"rig_name":"spoonbill_donovan_10f_6h","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"uid_log":"us_30171200_wb1_log_hfll_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30171200","log_delay":0,"id":1,"query_period":10,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h_old_575/log/us_30171200_wb1_log_hfll_time_1s","object":"log"},{"revision_waiting_time":0,"uid_log":"us_30171200_wb1_log_dfr_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30171200","log_delay":0,"id":2,"query_period":10,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h_old_575/log/us_30171200_wb1_log_dfr_time_1s","object":"log"},{"revision_waiting_time":0,"offset":0,"uid_log":"us_30171200_wb1_log_1","filters":{},"enabled":false,"index_type":"measured depth","uid_well":"us_30171200","id":3,"query_period":30,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h_old_575/log/us_30171200_wb1_log_1","object":"log"},{"uid_log":"*","filters":{},"enabled":false,"uid_well":"us_30171200","id":4,"query_period":600,"uid_wellbore":"us_30171200_wb1","state_path":"spoonbill-donovan_10f_6h_old_575/message","object":"message"}],"enabled":false,"mode":"client","protocol_version":"1.3.1.1","name":"SPOONBILL-DONOVAN 10F 6H (OLD 5.7.5)","id":7,"rig_name":"oldest_spoonbill_donovan_10f_6h","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"offset":300,"uid_log":"us_30125466_wb1_log_1","filters":{},"enabled":false,"index_type":"measured depth","uid_well":"us_30125466","id":1,"query_period":30,"uid_wellbore":"us_30125466_wb1","state_path":"pason_sealy_george_n503h_wb_1_nabors_x34/log/us_30125466_wb1_log_1","object":"log"},{"revision_waiting_time":0,"uid_log":"us_30125466_wb1_log_hfll_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30125466","log_delay":0,"id":2,"query_period":10,"uid_wellbore":"us_30125466_wb1","state_path":"pason_sealy_george_n503h_wb_1_nabors_x34/log/us_30125466_wb1_log_hfll_time_1s","object":"log"},{"revision_waiting_time":0,"uid_log":"us_30125466_wb1_log_dfr_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30125466","log_delay":0,"id":3,"query_period":10,"uid_wellbore":"us_30125466_wb1","state_path":"pason_sealy_george_n503h_wb_1_nabors_x34/log/us_30125466_wb1_log_dfr_time_1s","object":"log"}],"enabled":false,"mode":"client","protocol_version":"1.3.1.1","name":"Pason Sealy George N503H WB 1 (Nabors X34)","id":8,"rig_name":"sealy_george_n503h","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"uid_log":"us_30125466_wb1_log_dfr_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30125466","log_delay":0,"id":1,"query_period":10,"uid_wellbore":"us_30125466_wb1","state_path":"newest_sealy_george_n503h/log/us_30125466_wb1_log_dfr_time_1s","object":"log"}],"enabled":false,"mode":"client","protocol_version":"1.3.1.1","name":"Pason Sealy (Nabors X34) Backup","id":9,"rig_name":"newest_sealy_george_n503h","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"offset":0,"uid_log":"us_30125466_wb1_log_dfr_time_1s","filters":{},"enabled":true,"index_type":"date time","uid_well":"us_30125466","log_delay":0,"id":1,"query_period":20,"uid_wellbore":"us_30125466_wb1","state_path":"pason_sealy_nabors_x34_backup_2/log/us_30125466_wb1_log_dfr_time_1s","object":"log"}],"enabled":false,"mode":"client","protocol_version":"1.3.1.1","name":"Pason Sealy (Nabors X34) Backup 2","id":10,"rig_name":"5_7_6_sealy_george_n503h","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"offset":300,"uid_log":"us_30125466_wb1_log_1","filters":{},"enabled":false,"index_type":"measured depth","uid_well":"us_30125466","id":1,"query_period":30,"uid_wellbore":"us_30125466_wb1","state_path":"heavy_pason_sealy_george_n503h_wb_1_nabors_x34/log/us_30125466_wb1_log_1","object":"log"},{"revision_waiting_time":0,"uid_log":"us_30125466_wb1_log_hfll_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30125466","log_delay":0,"id":2,"query_period":10,"uid_wellbore":"us_30125466_wb1","state_path":"heavy_pason_sealy_george_n503h_wb_1_nabors_x34/log/us_30125466_wb1_log_hfll_time_1s","object":"log"},{"revision_waiting_time":0,"uid_log":"us_30125466_wb1_log_dfr_time_1s","filters":{},"enabled":false,"index_type":"date time","uid_well":"us_30125466","log_delay":0,"id":3,"query_period":10,"uid_wellbore":"us_30125466_wb1","state_path":"heavy_pason_sealy_george_n503h_wb_1_nabors_x34/log/us_30125466_wb1_log_dfr_time_1s","object":"log"}],"enabled":false,"mode":"client","protocol_version":"1.3.1.1","name":"Pason Sealy Thirdy (Nabors X34)","id":11,"rig_name":"yet_another_heavy_sealy_george_n503h","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Intelie","filters":{},"requests":[],"enabled":false,"mode":"client","protocol_version":"1.4.1.1","name":"Intelie STORE","id":12,"rig_name":"intelie_store","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Bardarsz","filters":{},"requests":[{"uid_log":"4e85ba06-fd0f-47f5-850a-79ebeb878bae","filters":{},"enabled":true,"index_type":"date time","uid_well":"d3a2c17f-dc9c-4d40-99ec-8a06072652d4","log_delay":0,"id":1,"uid_wellbore":"bc175de6-bafc-4867-a273-3b79f2ee3ee9","state_path":"bardarsz/log/4e85ba06-fd0f-47f5-850a-79ebeb878bae","object":"log"}],"enabled":false,"mode":"client","protocol_version":"1.4.1.1","name":"Bardarsz","id":13,"rig_name":"bardarsz","protocol_name":"witsml"},{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"offset":0,"uid_log":"us_1759454946_wb1_log_hfll_time_1s","filters":{},"enabled":true,"index_type":"date time","uid_well":"us_1759454946","log_delay":0,"id":1,"query_period":10,"uid_wellbore":"us_1759454946_wb1","state_path":"pason/log/us_1759454946_wb1_log_hfll_time_1s","object":"log"}],"enabled":true,"mode":"client","protocol_version":"1.3.1.1","name":"Pason","id":14,"rig_name":"pason_v1311","protocol_name":"witsml"}]},"runningConfig":{"sources":[{"tls_auth":false,"service_company":"Pason","filters":{},"requests":[{"revision_waiting_time":0,"offset":0,"uid_log":"us_1759454946_wb1_log_hfll_time_1s","filters":{},"enabled":true,"index_type":"date time","uid_well":"us_1759454946","log_delay":0,"id":1,"query_period":10,"uid_wellbore":"us_1759454946_wb1","state_path":"pason/log/us_1759454946_wb1_log_hfll_time_1s","object":"log"}],"enabled":true,"mode":"client","protocol_version":"1.3.1.1","name":"Pason","id":14,"rig_name":"pason_v1311","protocol_name":"witsml"}]},"system":{"disk":{"data":{"avail":"42722529280","total":"104552390656"}},"ntp":{"state":"ok","time":{"sec":"1761159859","usec":"73084199"},"maxerror":"710000","esterror":"0"}},"internalEventsPeriod":"60000"}

All the collected data is stored locally, on the collector machine’s disk, for 30 days. It guarantees that after any network issue, no data will be lost.