Skip to content

optimize memory usage for OleStream #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NiuBlibing wants to merge 2 commits into
base: master
Choose a base branch
from
Open

optimize memory usage for OleStream #171

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9da4b1e
optimize memory usage for OleStream
NiuBlibing Jan 24, 2025
79a4bf9
fix build error with python2
NiuBlibing Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 11 additions & 13 deletions olefile/olefile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,7 +642,7 @@ def __init__(self, fp, sect, size, offset, sectorsize, fat, filesize, olefileio)
# optimization(?): data is first a list of strings, and join() is called
# at the end to concatenate all in one string.
# (this may not be really useful with recent Python versions)
data = []
super(OleStream, self).__init__()
# if size is zero, then first sector index should be ENDOFCHAIN:
if size == 0 and sect != ENDOFCHAIN:
log.debug('size == 0 and sect != ENDOFCHAIN:')
Expand Down Expand Up @@ -692,7 +692,7 @@ def __init__(self, fp, sect, size, offset, sectorsize, fat, filesize, olefileio)
(sect, len(fat), offset+sectorsize*sect, filesize, len(sector_data)))
log.debug('seek+len(read)=%d' % (offset+sectorsize*sect+len(sector_data)))
self.ole._raise_defect(DEFECT_INCORRECT, 'incomplete OLE sector')
data.append(sector_data)
super(OleStream, self).write(sector_data)
# jump to next sector in the FAT:
try:
sect = fat[sect] & 0xFFFFFFFF # JYTHON-WORKAROUND
Expand All @@ -704,27 +704,25 @@ def __init__(self, fp, sect, size, offset, sectorsize, fat, filesize, olefileio)
# [PL] Last sector should be a "end of chain" marker:
# if sect != ENDOFCHAIN:
# raise IOError('incorrect last sector index in OLE stream')
data = b"".join(data)
# Data is truncated to the actual stream size:
if len(data) >= size:
log.debug('Read data of length %d, truncated to stream size %d' % (len(data), size))
data = data[:size]
data_len = super(OleStream, self).tell()
if data_len >= size:
log.debug('Read data of length %d, truncated to stream size %d' % (data_len, size))
super(OleStream, self).truncate(size)
# actual stream size is stored for future use:
self.size = size
elif unknown_size:
# actual stream size was not known, now we know the size of read
# data:
log.debug('Read data of length %d, the stream size was unknown' % len(data))
self.size = len(data)
log.debug('Read data of length %d, the stream size was unknown' % data_len)
self.size = data_len
else:
# read data is less than expected:
log.debug('Read data of length %d, less than expected stream size %d' % (len(data), size))
log.debug('Read data of length %d, less than expected stream size %d' % (data_len, size))
# TODO: provide details in exception message
self.size = len(data)
self.size = data_len
self.ole._raise_defect(DEFECT_INCORRECT, 'OLE stream size is less than declared')
# when all data is read in memory, BytesIO constructor is called
io.BytesIO.__init__(self, data)
# Then the OleStream object can be used as a read-only file object.
super(OleStream, self).seek(0)


# --- OleDirectoryEntry -------------------------------------------------------
Expand Down