Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/smmap/util.py: 38%

1"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" 

2import os 

3import sys 

4 

5from mmap import mmap, ACCESS_READ 

6from mmap import ALLOCATIONGRANULARITY 

7 

8__all__ = ["align_to_mmap", "is_64_bit", 

9 "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] 

10 

11#{ Utilities 

12 

13 

14def align_to_mmap(num, round_up): 

15 """ 

16 Align the given integer number to the closest page offset, which usually is 4096 bytes. 

17 

18 :param round_up: if True, the next higher multiple of page size is used, otherwise 

19 the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) 

20 :return: num rounded to closest page""" 

21 res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY 

22 if round_up and (res != num): 

23 res += ALLOCATIONGRANULARITY 

24 # END handle size 

25 return res 

26 

27 

28def is_64_bit(): 

29 """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" 

30 return sys.maxsize > (1 << 32) - 1 

31 

32#}END utilities 

33 

34 

35#{ Utility Classes 

36 

37class MapWindow: 

38 

39 """Utility type which is used to snap windows towards each other, and to adjust their size""" 

40 __slots__ = ( 

41 'ofs', # offset into the file in bytes 

42 'size' # size of the window in bytes 

43 ) 

44 

45 def __init__(self, offset, size): 

46 self.ofs = offset 

47 self.size = size 

48 

49 def __repr__(self): 

50 return "MapWindow(%i, %i)" % (self.ofs, self.size) 

51 

52 @classmethod 

53 def from_region(cls, region): 

54 """:return: new window from a region""" 

55 return cls(region._b, region.size()) 

56 

57 def ofs_end(self): 

58 return self.ofs + self.size 

59 

60 def align(self): 

61 """Assures the previous window area is contained in the new one""" 

62 nofs = align_to_mmap(self.ofs, 0) 

63 self.size += self.ofs - nofs # keep size constant 

64 self.ofs = nofs 

65 self.size = align_to_mmap(self.size, 1) 

66 

67 def extend_left_to(self, window, max_size): 

68 """Adjust the offset to start where the given window on our left ends if possible, 

69 but don't make yourself larger than max_size. 

70 The resize will assure that the new window still contains the old window area""" 

71 rofs = self.ofs - window.ofs_end() 

72 nsize = rofs + self.size 

73 rofs -= nsize - min(nsize, max_size) 

74 self.ofs = self.ofs - rofs 

75 self.size += rofs 

76 

77 def extend_right_to(self, window, max_size): 

78 """Adjust the size to make our window end where the right window begins, but don't 

79 get larger than max_size""" 

80 self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) 

81 

82 

83class MapRegion: 

84 

85 """Defines a mapped region of memory, aligned to pagesizes 

86 

87 **Note:** deallocates used region automatically on destruction""" 

88 __slots__ = [ 

89 '_b', # beginning of mapping 

90 '_mf', # mapped memory chunk (as returned by mmap) 

91 '_uc', # total amount of usages 

92 '_size', # cached size of our memory map 

93 '__weakref__' 

94 ] 

95 

96 #{ Configuration 

97 #} END configuration 

98 

99 def __init__(self, path_or_fd, ofs, size, flags=0): 

100 """Initialize a region, allocate the memory map 

101 :param path_or_fd: path to the file to map, or the opened file descriptor 

102 :param ofs: **aligned** offset into the file to be mapped 

103 :param size: if size is larger then the file on disk, the whole file will be 

104 allocated the the size automatically adjusted 

105 :param flags: additional flags to be given when opening the file. 

106 :raise Exception: if no memory can be allocated""" 

107 self._b = ofs 

108 self._size = 0 

109 self._uc = 0 

110 

111 if isinstance(path_or_fd, int): 

112 fd = path_or_fd 

113 else: 

114 fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) 

115 # END handle fd 

116 

117 try: 

118 kwargs = dict(access=ACCESS_READ, offset=ofs) 

119 corrected_size = size 

120 sizeofs = ofs 

121 

122 # have to correct size, otherwise (instead of the c version) it will 

123 # bark that the size is too large ... many extra file accesses because 

124 # if this ... argh ! 

125 actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) 

126 self._mf = mmap(fd, actual_size, **kwargs) 

127 # END handle memory mode 

128 

129 self._size = len(self._mf) 

130 finally: 

131 if isinstance(path_or_fd, str): 

132 os.close(fd) 

133 # END only close it if we opened it 

134 # END close file handle 

135 # We assume the first one to use us keeps us around 

136 self.increment_client_count() 

137 

138 def __repr__(self): 

139 return "MapRegion<%i, %i>" % (self._b, self.size()) 

140 

141 #{ Interface 

142 

143 def buffer(self): 

144 """:return: a buffer containing the memory""" 

145 return self._mf 

146 

147 def map(self): 

148 """:return: a memory map containing the memory""" 

149 return self._mf 

150 

151 def ofs_begin(self): 

152 """:return: absolute byte offset to the first byte of the mapping""" 

153 return self._b 

154 

155 def size(self): 

156 """:return: total size of the mapped region in bytes""" 

157 return self._size 

158 

159 def ofs_end(self): 

160 """:return: Absolute offset to one byte beyond the mapping into the file""" 

161 return self._b + self._size 

162 

163 def includes_ofs(self, ofs): 

164 """:return: True if the given offset can be read in our mapped region""" 

165 return self._b <= ofs < self._b + self._size 

166 

167 def client_count(self): 

168 """:return: number of clients currently using this region""" 

169 return self._uc 

170 

171 def increment_client_count(self, ofs = 1): 

172 """Adjust the usage count by the given positive or negative offset. 

173 If usage count equals 0, we will auto-release our resources 

174 :return: True if we released resources, False otherwise. In the latter case, we can still be used""" 

175 self._uc += ofs 

176 assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc 

177 

178 if self.client_count() == 0: 

179 self.release() 

180 return True 

181 else: 

182 return False 

183 # end handle release 

184 

185 def release(self): 

186 """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" 

187 self._mf.close() 

188 

189 #} END interface 

190 

191 

192class MapRegionList(list): 

193 

194 """List of MapRegion instances associating a path with a list of regions.""" 

195 __slots__ = ( 

196 '_path_or_fd', # path or file descriptor which is mapped by all our regions 

197 '_file_size' # total size of the file we map 

198 ) 

199 

200 def __new__(cls, path): 

201 return super().__new__(cls) 

202 

203 def __init__(self, path_or_fd): 

204 self._path_or_fd = path_or_fd 

205 self._file_size = None 

206 

207 def path_or_fd(self): 

208 """:return: path or file descriptor we are attached to""" 

209 return self._path_or_fd 

210 

211 def file_size(self): 

212 """:return: size of file we manager""" 

213 if self._file_size is None: 

214 if isinstance(self._path_or_fd, str): 

215 self._file_size = os.stat(self._path_or_fd).st_size 

216 else: 

217 self._file_size = os.fstat(self._path_or_fd).st_size 

218 # END handle path type 

219 # END update file size 

220 return self._file_size 

221 

222#} END utility classes