We have updated our Terms of Service, Code of Conduct, and Addendum.

Unable to get collection tasks to run after Discover on REST API Collector

JH Liew
JH Liew Posts: 13 ✭
in Stream

I have a REST API Collector which look at retrieving a number of records containing an ID which then need to be fed into a collection task as input for a 2-stage API call.

While the Discover task has successfully retrieved the records with the ID, and I can see that in debug log the collection task has pick up the ID and insert correctly into Collect URI using variable, the tasks simply get cancelled afterwards.

Does anyone have experience resolving similar issue before or have some idea what this might have gone wrong?

Comments

  • JH Liew
    JH Liew Posts: 13 ✭
    edited June 21

    This is how the log looks like for the collection tasks come after the discovery:

    [
    {
    "time": "2024-06-21T06:16:35.833Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "cancelled all in flight tasks",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    },
    {
    "time": "2024-06-21T06:16:35.752Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "cancelling in flight tasks",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "numTask": 1,
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    },
    {
    "time": "2024-06-21T06:16:35.751Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "previousState": "running",
    "currentState": "cancelled",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    },
    {
    "time": "2024-06-21T06:16:35.741Z",
    "cid": "w1",
    "channel": "TaskExecutor",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "previousState": "running",
    "currentState": "cancelled",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/task/tasklog.log"
    },
    {
    "time": "2024-06-21T06:16:35.741Z",
    "cid": "w1",
    "channel": "task:1",
    "level": "info",
    "message": "cancelling task",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/task/tasklog.log"
    },
    {
    "time": "2024-06-21T06:16:35.402Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "previousState": "pending",
    "currentState": "running",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    },
    {
    "time": "2024-06-21T06:16:35.401Z",
    "cid": "w1",
    "channel": "TaskExecutor",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "previousState": "pending",
    "currentState": "running",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/task/tasklog.log"
    },
    {
    "time": "2024-06-21T06:16:35.400Z",
    "cid": "w1",
    "channel": "TaskExecutor",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "previousState": "initializing",
    "currentState": "pending",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/task/tasklog.log"
    },
    {
    "time": "2024-06-21T06:16:35.391Z",
    "cid": "w1",
    "channel": "TaskExecutor",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "currentState": "initializing",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/task/tasklog.log"
    },
    {
    "time": "2024-06-21T06:16:35.385Z",
    "cid": "w1",
    "channel": "task",
    "level": "info",
    "message": "Task starting",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "dir": "/opt/cribl/config-volume",
    "executor": {
    "type": "filesystem_search",
    "heartbeatPeriod": 60
    },
    "task": {
    "type": "filesystem_search",
    "conf": {
    "path": "$CRIBL_HOME/state/jobs/default/1718950579.13592.adhoc.MyRESTCollector/tasks/${taskId}/${resultType}/${filename}",
    "filter": "resultType == 'log'",
    "contentFilter": "true && (!_time || (1718950519161 <= _time && _time <= 1718950595127))",
    "maxResults": 1000,
    "reverseResults": true,
    "resultBatchSize": 100,
    "compareExpression": "evtB._time - evtA._time"
    }
    },
    "logLevel": "info",
    "pid": 636718,
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/task/tasklog.log"
    },
    {
    "time": "2024-06-21T06:16:35.299Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "created task manifest reader",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "file": {
    "_events": {},
    "_eventsCount": 0
    },
    "bufferSize": 4096,
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    },
    {
    "time": "2024-06-21T06:16:35.298Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "previousState": "initializing",
    "currentState": "pending",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    },
    {
    "time": "2024-06-21T06:16:35.293Z",
    "cid": "api",
    "channel": "Job",
    "level": "info",
    "message": "execution state change",
    "jobId": "1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector",
    "currentState": "initializing",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1718950595.13593.system.fetch-job-logs-1718950579.13592.adhoc.MyRESTCollector/logs/job/job.log"
    }
    ]

  • JH Liew
    JH Liew Posts: 13 ✭

    Currently I am only able to set Discover job to capture Debug logs, even under Full run. Is there a way to also set Debug logging for the individual collection tasks to see what's going on?

    Thanks.

  • JH Liew
    JH Liew Posts: 13 ✭

    Never mind about Debug log, I managed to figure it out. Looking through the logs, I have obtain some additional details:

    {
    "time": "2024-06-24T05:58:11.182Z",
    "cid": "w0",
    "channel": "task:1",
    "level": "debug",
    "message": "filtered out file",
    "jobId": "1719208691.17630.system.fetch-job-logs-1719208500.17626.scheduled.MyRESTCollector",
    "taskId": "1",
    "host": "MyHost",
    "file": "/opt/cribl/state/jobs/default/1719208500.17626.scheduled.MyRESTCollector/tasks/discover/keep",
    "reason": "failed iterator file filter",
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1719208691.17630.system.fetch-job-logs-1719208500.17626.scheduled.MyRESTCollector/logs/task/tasklog.log"
    }

    Not sure if I understand, were the reason ("failed iterator file filter") for the cancelled jobs related to the conditions set for scheduling of the collector tasks? How can I configure this correctly?

    {
    "time": "2024-06-24T05:58:11.131Z",
    "cid": "api",
    "channel": "Job",
    "level": "debug",
    "message": "added task",
    "jobId": "1719208691.17630.system.fetch-job-logs-1719208500.17626.scheduled.MyRESTCollector",
    "task": {
    "jobId": "1719208691.17630.system.fetch-job-logs-1719208500.17626.scheduled.MyRESTCollector",
    "taskId": "1",
    "dir": "/opt/cribl/config-volume",
    "executor": {
    "type": "filesystem_search",
    "heartbeatPeriod": 60
    },
    "task": {
    "type": "filesystem_search",
    "conf": {
    "path": "$CRIBL_HOME/state/jobs/default/1719208500.17626.scheduled.MyRESTCollector/tasks/${taskId}/${resultType}/${filename}",
    "filter": "resultType == 'log'",
    "contentFilter": "true && (!_time || (1719208440580 <= _time && _time <= 1719208690915))",
    "maxResults": 1000,
    "reverseResults": true,
    "resultBatchSize": 100,
    "compareExpression": "evtB._time - evtA._time"
    }
    },
    "logLevel": "debug"
    },
    "source": "/opt/cribl/config-volume/state/jobs/MyGroup/1719208691.17630.system.fetch-job-logs-1719208500.17626.scheduled.MyRESTCollector/logs/job/job.log"
    }'

Categories