XRootD
Loading...
Searching...
No Matches
XrdPfcInfo.hh
Go to the documentation of this file.
1#ifndef __XRDPFC_INFO_HH__
2#define __XRDPFC_INFO_HH__
3//----------------------------------------------------------------------------------
4// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6//----------------------------------------------------------------------------------
7// XRootD is free software: you can redistribute it and/or modify
8// it under the terms of the GNU Lesser General Public License as published by
9// the Free Software Foundation, either version 3 of the License, or
10// (at your option) any later version.
11//
12// XRootD is distributed in the hope that it will be useful,
13// but WITHOUT ANY WARRANTY; without even the implied warranty of
14// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15// GNU General Public License for more details.
16//
17// You should have received a copy of the GNU Lesser General Public License
18// along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19//----------------------------------------------------------------------------------
20
21#include "XrdPfcTypes.hh"
22
23#include <cstdio>
24#include <ctime>
25#include <cassert>
26#include <vector>
27
28class XrdOssDF;
29class XrdCksCalc;
30class XrdSysTrace;
31
32namespace XrdPfc
33{
34class Stats;
35
36//----------------------------------------------------------------------------
38//----------------------------------------------------------------------------
39
40class Info
41{
42public:
43 struct Status {
44 union {
45 struct {
46 int f_cksum_check : 3;
47
48 int _free_bits_ : 29;
49 };
50 unsigned int _raw_;
51 };
52 Status() : _raw_(0) {}
53 };
54
56 struct AStat
57 {
58 time_t AttachTime;
59 time_t DetachTime;
60 int NumIos;
64 long long BytesHit;
65 long long BytesMissed;
66 long long BytesBypassed;
67
71 {}
72
73 void MergeWith(const AStat &a);
74 };
75
91
92
93 //------------------------------------------------------------------------
95 //------------------------------------------------------------------------
96 Info(XrdSysTrace* trace, bool prefetchBuffer = false);
97
98 //------------------------------------------------------------------------
100 //------------------------------------------------------------------------
101 ~Info();
102
103 //---------------------------------------------------------------------
105 //---------------------------------------------------------------------
106 void SetBitWritten(int i);
107
108 //---------------------------------------------------------------------
110 //---------------------------------------------------------------------
111 bool TestBitWritten(int i) const;
112
113 //---------------------------------------------------------------------
115 //---------------------------------------------------------------------
116 bool TestBitPrefetch(int i) const;
117
118 //---------------------------------------------------------------------
120 //---------------------------------------------------------------------
121 void SetBitPrefetch(int i);
122
123 //---------------------------------------------------------------------
125 //---------------------------------------------------------------------
126 void SetBitSynced(int i);
127
128 //---------------------------------------------------------------------
130 //---------------------------------------------------------------------
131 void SetAllBitsSynced();
132
133 void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs);
134
135 //---------------------------------------------------------------------
137 //---------------------------------------------------------------------
138 void ResizeBits();
139
140 //---------------------------------------------------------------------
146 //---------------------------------------------------------------------
147 bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0);
148
149 //---------------------------------------------------------------------
155 //---------------------------------------------------------------------
156 bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0);
157
158 //---------------------------------------------------------------------
160 //---------------------------------------------------------------------
162
163 //---------------------------------------------------------------------
165 //---------------------------------------------------------------------
166 void ResetAllAccessStats();
167
168 //---------------------------------------------------------------------
170 //---------------------------------------------------------------------
171 void WriteIOStatAttach();
172
173 //---------------------------------------------------------------------
175 //---------------------------------------------------------------------
176 void WriteIOStat(Stats& s);
177
178 //---------------------------------------------------------------------
180 //---------------------------------------------------------------------
181 void WriteIOStatDetach(Stats& s);
182
183 //---------------------------------------------------------------------
185 //---------------------------------------------------------------------
186 void WriteIOStatSingle(long long bytes_disk);
187
188 //---------------------------------------------------------------------
190 //---------------------------------------------------------------------
191 void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
192
193 //---------------------------------------------------------------------
195 //---------------------------------------------------------------------
196 int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const;
197
198 //---------------------------------------------------------------------
200 //---------------------------------------------------------------------
201 int GetBitvecSizeInBytes() const;
202
203 //---------------------------------------------------------------------
205 //---------------------------------------------------------------------
206 int GetNBlocks() const;
207
208 //---------------------------------------------------------------------
210 //---------------------------------------------------------------------
211 long long GetFileSize() const;
212
213 //---------------------------------------------------------------------
215 //---------------------------------------------------------------------
216 bool GetLatestDetachTime(time_t& t) const;
217
218 //---------------------------------------------------------------------
220 //---------------------------------------------------------------------
221 const AStat* GetLastAccessStats() const;
222
223 //---------------------------------------------------------------------
225 //---------------------------------------------------------------------
226 long long GetBufferSize() const;
227
228 //---------------------------------------------------------------------
230 //---------------------------------------------------------------------
231 bool IsComplete() const;
232
233 //---------------------------------------------------------------------
235 //---------------------------------------------------------------------
236 int GetNDownloadedBlocks() const;
237
238 //---------------------------------------------------------------------
240 //---------------------------------------------------------------------
241 long long GetNDownloadedBytes() const;
242
243 //---------------------------------------------------------------------
245 //---------------------------------------------------------------------
246 int GetLastDownloadedBlock() const;
247
248 //---------------------------------------------------------------------
250 //---------------------------------------------------------------------
251 long long GetExpectedDataFileSize() const;
252
253 //---------------------------------------------------------------------
255 //---------------------------------------------------------------------
257
258 //---------------------------------------------------------------------
260 //---------------------------------------------------------------------
261 size_t GetAccessCnt() const { return m_store.m_accessCnt; }
262
263 //---------------------------------------------------------------------
265 //---------------------------------------------------------------------
266 int GetVersion() { return m_version; }
267
268 //---------------------------------------------------------------------
270 //---------------------------------------------------------------------
271 const Store& RefStoredData() const { return m_store; }
272 const std::vector<AStat>& RefAStats() const { return m_astats; }
273
274 //---------------------------------------------------------------------
276 //---------------------------------------------------------------------
277 time_t GetCreationTime() const { return m_store.m_creationTime; }
278
279 //---------------------------------------------------------------------
281 //---------------------------------------------------------------------
282 uint32_t CalcCksumStore();
283 uint32_t CalcCksumSyncedAndAStats();
284 void CalcCksumMd5(unsigned char* buff, char* digest);
285
286 CkSumCheck_e GetCkSumState() const { return (CkSumCheck_e) m_store.m_status.f_cksum_check; }
287 const char* GetCkSumStateAsText() const;
288
289 bool IsCkSumCache() const { return m_store.m_status.f_cksum_check & CSChk_Cache; }
290 bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; }
291 bool IsCkSumAny() const { return m_store.m_status.f_cksum_check & CSChk_Both; }
292 bool IsCkSumBoth() const { return (m_store.m_status.f_cksum_check & CSChk_Both) == CSChk_Both; }
293
294 void SetCkSumState(CkSumCheck_e css) { m_store.m_status.f_cksum_check = css; }
295 void DowngradeCkSumState(CkSumCheck_e css_ref) { m_store.m_status.f_cksum_check &= css_ref; }
296 void ResetCkSumCache();
297 void ResetCkSumNet();
298
299 bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; }
300 time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; }
303
304#ifdef XRDPFC_CKSUM_TEST
305 static void TestCksumStuff();
306#endif
307
308 static const char* m_traceID; // has to be m_ (convention in TRACE macros)
309 static const char* s_infoExtension;
310 static const size_t s_infoExtensionLen;
311 static size_t s_maxNumAccess; // can be set from configuration
312 static const int s_defaultVersion;
313
314 XrdSysTrace* GetTrace() const {return m_trace; }
315
316protected:
318
320 unsigned char *m_buff_synced;
321 unsigned char *m_buff_written;
322 unsigned char *m_buff_prefetch;
323 std::vector<AStat> m_astats;
324
330
331private:
332 inline unsigned char cfiBIT(int n) const { return 1 << n; }
333
334 // Reading functions for older cinfo file formats
335 bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
336 bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
337
338 XrdCksCalc* m_cksCalcMd5;
339};
340
341//------------------------------------------------------------------------------
342
343inline bool Info::TestBitWritten(int i) const
344{
345 const int cn = i/8;
346 assert(cn < GetBitvecSizeInBytes());
347
348 const int off = i - cn*8;
349 return (m_buff_written[cn] & cfiBIT(off)) != 0;
350}
351
352inline void Info::SetBitWritten(int i)
353{
354 const int cn = i/8;
355 assert(cn < GetBitvecSizeInBytes());
356
357 const int off = i - cn*8;
358
359 m_buff_written[cn] |= cfiBIT(off);
360
361 if (--m_missingBlocks == 0)
362 m_complete = true;
363}
364
365inline void Info::SetBitPrefetch(int i)
366{
367 if (!m_buff_prefetch) return;
368
369 const int cn = i/8;
370 assert(cn < GetBitvecSizeInBytes());
371
372 const int off = i - cn*8;
373 m_buff_prefetch[cn] |= cfiBIT(off);
374}
375
376inline bool Info::TestBitPrefetch(int i) const
377{
378 if (!m_buff_prefetch) return false;
379
380 const int cn = i/8;
381 assert(cn < GetBitvecSizeInBytes());
382
383 const int off = i - cn*8;
384 return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
385}
386
387inline void Info::SetBitSynced(int i)
388{
389 const int cn = i/8;
390 assert(cn < GetBitvecSizeInBytes());
391
392 const int off = i - cn*8;
393 m_buff_synced[cn] |= cfiBIT(off);
394}
395
396//------------------------------------------------------------------------------
397
399{
400 int cntd = 0;
401 for (int i = 0; i < m_bitvecSizeInBits; ++i)
402 if (TestBitWritten(i)) cntd++;
403
404 return cntd;
405}
406
407inline long long Info::GetNDownloadedBytes() const
408{
410}
411
413{
414 for (int i = m_bitvecSizeInBits - 1; i >= 0; --i)
415 if (TestBitWritten(i)) return i;
416
417 return -1;
418}
419
420inline long long Info::GetExpectedDataFileSize() const
421{
422 int last_block = GetLastDownloadedBlock();
423 if (last_block == m_bitvecSizeInBits - 1)
424 return m_store.m_file_size;
425 else
426 return (last_block + 1) * m_store.m_buffer_size;
427}
428
430{
432 return ((m_bitvecSizeInBits - 1)/8 + 1);
433 else
434 return 0;
435}
436
437inline int Info::GetNBlocks() const
438{
439 return m_bitvecSizeInBits;
440}
441
442inline long long Info::GetFileSize() const
443{
444 return m_store.m_file_size;
445}
446
447inline bool Info::IsComplete() const
448{
449 return m_complete;
450}
451
452inline int Info::CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
453{
454 // TODO rewrite to use full byte comparisons outside of edges ?
455 // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits.
456 int cnt = 0;
457 for (int i = firstIdx; i < lastIdx; ++i)
458 if (! TestBitWritten(i)) ++cnt;
459
460 return cnt;
461}
462
468
469inline long long Info::GetBufferSize() const
470{
471 return m_store.m_buffer_size;
472}
473
474}
475#endif
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:41
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
void UpdateDownloadCompleteStatus()
Update complete status.
bool IsCkSumAny() const
static const char * s_infoExtension
uint32_t CalcCksumStore()
Get cksum, MD5 is for backward compatibility with V2 and V3.
void ResizeBits()
Reserve bit vectors for file_size / buffer_size bytes.
void ResetCkSumCache()
static const int s_defaultVersion
const std::vector< AStat > & RefAStats() const
void SetBitSynced(int i)
Mark block as synced to disk.
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
int GetBitvecSizeInBytes() const
Get size of download-state bit-vector in bytes.
time_t GetNoCkSumTimeForUVKeep() const
unsigned char * m_buff_prefetch
prefetch statistics
static const size_t s_infoExtensionLen
CkSumCheck_e GetCkSumState() const
const AStat * GetLastAccessStats() const
Get latest access stats.
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
void ResetCkSumNet()
bool Write(XrdOssDF *fp, const char *dname, const char *fname=0)
void DowngradeCkSumState(CkSumCheck_e css_ref)
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
XrdSysTrace * GetTrace() const
const Store & RefStoredData() const
Get stored data.
uint32_t CalcCksumSyncedAndAStats()
~Info()
Destructor.
time_t GetCreationTime() const
Get file size.
unsigned char * m_buff_written
download state vector
bool IsCkSumNet() const
unsigned char * m_buff_synced
disk written state vector
bool m_complete
cached; if false, set to true when missingBlocks hit zero
void ResetAllAccessStats()
Reset IO Stats.
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
bool IsComplete() const
Get complete status.
bool IsCkSumCache() const
void SetBitWritten(int i)
Mark block as written to disk.
time_t GetNoCkSumTime() const
int m_missingBlocks
cached, updated in SetBitWritten()
long long GetBufferSize() const
Get prefetch buffer size.
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
long long GetExpectedDataFileSize() const
Get expected data file size.
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
int m_bitvecSizeInBits
cached
long long GetFileSize() const
Get file size.
XrdSysTrace * m_trace
std::vector< AStat > m_astats
access records
void SetAllBitsSynced()
Mark all blocks as synced to disk.
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
void SetCkSumState(CkSumCheck_e css)
const char * GetCkSumStateAsText() const
void ResetNoCkSumTime()
bool IsCkSumBoth() const
int GetVersion()
Get version.
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
size_t GetAccessCnt() const
Get number of accesses.
int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs)
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
static size_t s_maxNumAccess
static const char * m_traceID
void CalcCksumMd5(unsigned char *buff, char *digest)
int GetNBlocks() const
Get number of blocks represented in download-state bit-vector.
bool m_hasPrefetchBuffer
constains current prefetch score
bool HasNoCkSumTime() const
Statistics of cache utilisation by a File object.
@ CSChk_Cache
XrdPosixStats Stats
Access statistics.
Definition XrdPfcInfo.hh:57
long long BytesHit
read from cache
Definition XrdPfcInfo.hh:64
long long BytesBypassed
read from remote and dropped
Definition XrdPfcInfo.hh:66
void MergeWith(const AStat &a)
int Duration
total duration of all IOs attached
Definition XrdPfcInfo.hh:61
int Reserved
reserved / alignment
Definition XrdPfcInfo.hh:63
int NumIos
number of IO objects attached during this access
Definition XrdPfcInfo.hh:60
time_t DetachTime
close time
Definition XrdPfcInfo.hh:59
long long BytesMissed
read from remote and cached
Definition XrdPfcInfo.hh:65
time_t AttachTime
open time
Definition XrdPfcInfo.hh:58
int NumMerged
number of times the record has been merged
Definition XrdPfcInfo.hh:62
int m_astatSize
size of AStat vector
Definition XrdPfcInfo.hh:84
Status m_status
status information
Definition XrdPfcInfo.hh:83
long long m_buffer_size
buffer / block size
Definition XrdPfcInfo.hh:78
time_t m_noCkSumTime
time when first non-cksummed block was detected
Definition XrdPfcInfo.hh:81
size_t m_accessCnt
total access count for the file
Definition XrdPfcInfo.hh:82
long long m_file_size
size of file in bytes
Definition XrdPfcInfo.hh:79
time_t m_creationTime
time the info file was created
Definition XrdPfcInfo.hh:80