Lines Matching full:job
572 /* A future job may error and skip our job */ in ZSTDMT_serialState_update()
617 DEBUGLOG(5, "Skipping past job %u because of error", jobID); in ZSTDMT_serialState_ensureFinished()
660 ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \
661 job->cSize = e; \
662 ZSTD_pthread_mutex_unlock(&job->job_mutex); \
669 ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; in ZSTDMT_compressionJob() local
670 …ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the cop… in ZSTDMT_compressionJob()
671 ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); in ZSTDMT_compressionJob()
672 rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); in ZSTDMT_compressionJob()
673 buffer_t dstBuff = job->dstBuff; in ZSTDMT_compressionJob()
678 if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ in ZSTDMT_compressionJob()
679 dstBuff = ZSTDMT_getBuffer(job->bufPool); in ZSTDMT_compressionJob()
681 …job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job … in ZSTDMT_compressionJob()
689 if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; in ZSTDMT_compressionJob()
697 if (job->cdict) { in ZSTDMT_compressionJob()
698 …dvanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFr… in ZSTDMT_compressionJob()
699 assert(job->firstJob); /* only allowed for first job */ in ZSTDMT_compressionJob()
702 U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; in ZSTDMT_compressionJob()
703 …forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); in ZSTDMT_compressionJob()
706 if (!job->firstJob) { in ZSTDMT_compressionJob()
711 …job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mod… in ZSTDMT_compressionJob()
719 ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); in ZSTDMT_compressionJob()
721 if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ in ZSTDMT_compressionJob()
722 …size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, … in ZSTDMT_compressionJob()
724 …"ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); in ZSTDMT_compressionJob()
730 int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); in ZSTDMT_compressionJob()
731 const BYTE* ip = (const BYTE*) job->src.start; in ZSTDMT_compressionJob()
736 …if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* chec… in ZSTDMT_compressionJob()
737 …DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); in ZSTDMT_compressionJob()
738 assert(job->cSize == 0); in ZSTDMT_compressionJob()
745 ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); in ZSTDMT_compressionJob()
746 job->cSize += cSize; in ZSTDMT_compressionJob()
747 job->consumed = chunkSize * chunkNb; in ZSTDMT_compressionJob()
749 (U32)cSize, (U32)job->cSize); in ZSTDMT_compressionJob()
750 … ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ in ZSTDMT_compressionJob()
751 ZSTD_pthread_mutex_unlock(&job->job_mutex); in ZSTDMT_compressionJob()
756 if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { in ZSTDMT_compressionJob()
757 size_t const lastBlockSize1 = job->src.size & (chunkSize-1); in ZSTDMT_compressionJob()
758 …size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : last… in ZSTDMT_compressionJob()
759 size_t const cSize = (job->lastJob) ? in ZSTDMT_compressionJob()
765 if (!job->firstJob) { in ZSTDMT_compressionJob()
774 ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); in ZSTDMT_compressionJob()
775 if (job->prefix.size > 0) in ZSTDMT_compressionJob()
776 DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); in ZSTDMT_compressionJob()
777 DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); in ZSTDMT_compressionJob()
779 ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); in ZSTDMT_compressionJob()
780 ZSTDMT_releaseCCtx(job->cctxPool, cctx); in ZSTDMT_compressionJob()
782 ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); in ZSTDMT_compressionJob()
783 if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); in ZSTDMT_compressionJob()
784 job->cSize += lastCBlockSize; in ZSTDMT_compressionJob()
785 …job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presum… in ZSTDMT_compressionJob()
786 ZSTD_pthread_cond_signal(&job->job_cond); in ZSTDMT_compressionJob()
787 ZSTD_pthread_mutex_unlock(&job->job_mutex); in ZSTDMT_compressionJob()
804 * handles handing out job input buffers, and makes
843 …int jobReady; /* 1 => one job is already prepared, but pool has shortage of workers. Don't …
874 * allocate and init a job table.
899 if (nbJobs > mtctx->jobIDMask+1) { /* need more job capacity */ in ZSTDMT_expandJobsTable()
984 …DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.st… in ZSTDMT_releaseAllJobResources()
987 /* Clear the job description, but keep the mutex/cond */ in ZSTDMT_releaseAllJobResources()
1004 …DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to bloc… in ZSTDMT_waitForAllJobsCompleted()
1017 ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ in ZSTDMT_freeCCtx()
1063 * New parameters will be applied to next compression job. */
1120 if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ in ZSTDMT_toFlushNow()
1122 /* look into oldest non-fully-flushed job */ in ZSTDMT_toFlushNow()
1135 * ZSTDMT_flushProduced() should have already moved onto next job. in ZSTDMT_toFlushNow()
1269 /* Aim for the targetsectionSize as the average job size. */ in ZSTDMT_initCStream_internal()
1273 * expected job size is at least 4x larger. */ in ZSTDMT_initCStream_internal()
1280 …>targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= ove… in ZSTDMT_initCStream_internal()
1281 …DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)param… in ZSTDMT_initCStream_internal()
1330 * Job must be created from streaming variant.
1333 static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) in ZSTDMT_writeLastEmptyBlock() argument
1335 assert(job->lastJob == 1); in ZSTDMT_writeLastEmptyBlock()
1336 …assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ in ZSTDMT_writeLastEmptyBlock()
1337 assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ in ZSTDMT_writeLastEmptyBlock()
1338 …assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff m… in ZSTDMT_writeLastEmptyBlock()
1339 job->dstBuff = ZSTDMT_getBuffer(job->bufPool); in ZSTDMT_writeLastEmptyBlock()
1340 if (job->dstBuff.start == NULL) { in ZSTDMT_writeLastEmptyBlock()
1341 job->cSize = ERROR(memory_allocation); in ZSTDMT_writeLastEmptyBlock()
1344 … assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ in ZSTDMT_writeLastEmptyBlock()
1345 job->src = kNullRange; in ZSTDMT_writeLastEmptyBlock()
1346 job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); in ZSTDMT_writeLastEmptyBlock()
1347 assert(!ZSTD_isError(job->cSize)); in ZSTDMT_writeLastEmptyBlock()
1348 assert(job->consumed == 0); in ZSTDMT_writeLastEmptyBlock()
1357 DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); in ZSTDMT_createCompressionJob()
1364 … DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", in ZSTDMT_createCompressionJob()
1399 … /* single job exception : checksum is already calculated directly within worker thread */ in ZSTDMT_createCompressionJob()
1404 && (mtctx->nextJobID>0)/*single job must also write frame header*/ ) { in ZSTDMT_createCompressionJob()
1413 …DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u)… in ZSTDMT_createCompressionJob()
1423 … DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); in ZSTDMT_createCompressionJob()
1431 * flush whatever data has been produced but not yet flushed in current job.
1432 * move to next job if current one is fully flushed.
1439 DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", in ZSTDMT_flushProduced()
1449 … DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", in ZSTDMT_flushProduced()
1453 DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", in ZSTDMT_flushProduced()
1464 DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", in ZSTDMT_flushProduced()
1472 if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ in ZSTDMT_flushProduced()
1484 …DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", in ZSTDMT_flushProduced()
1497 if ( (srcConsumed == srcSize) /* job is completed */ in ZSTDMT_flushProduced()
1498 …jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ in ZSTDMT_flushProduced()
1499 DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", in ZSTDMT_flushProduced()
1504 …mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future che… in ZSTDMT_flushProduced()
1512 if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ in ZSTDMT_flushProduced()
1515 if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ in ZSTDMT_flushProduced()
1516 …inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ in ZSTDMT_flushProduced()
1523 * Returns the range of data used by the earliest job that is not yet complete.
1524 * If the data of the first job is broken up into two segments, we cover both
1547 /* Job source in multiple segments not supported yet */ in ZSTDMT_getInputDataInUse()
1748 * This likely happened because the job table was full so we in findSynchronizationPoint()
1749 * couldn't add our job. in findSynchronizationPoint()
1758 * job off, and tell the compressor to flush the job. Otherwise, load in findSynchronizationPoint()
1836 … * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. in ZSTDMT_compressStream_generic()
1837 * - We filled the input buffer: flush this job but don't end the frame. in ZSTDMT_compressStream_generic()
1838 * - We hit a synchronization point: flush this job but don't end the frame. in ZSTDMT_compressStream_generic()