XRootD
Loading...
Searching...
No Matches
XrdPfc Namespace Reference

Classes

class  Block
 
class  BlockResponseHandler
 
class  Cache
 Attaches/creates and detaches/deletes cache-io objects for disk based cache. More...
 
struct  ChunkRequest
 
struct  Configuration
 Contains parameters configurable from the xrootd config file. More...
 
struct  DataFsPurgeshot
 
struct  DataFsSnapshot
 
struct  DataFsState
 
struct  DataFsStateBase
 
class  Decision
 Base class for selecting which files should be cached. More...
 
class  DirectResponseHandler
 
struct  DirPurgeElement
 
struct  DirState
 
struct  DirStateBase
 
struct  DirStateElement
 
class  DirStats
 
struct  DirUsage
 
class  File
 
class  FPurgeState
 
class  FsTraversal
 
class  Info
 Status of cached file. Can be read from and written into a binary file. More...
 
class  IO
 Base cache-io class that implements some XrdOucCacheIO abstract methods. More...
 
class  IOFile
 Downloads original file into a single file on local disk. Handles read requests as they come along. More...
 
class  IOFileBlock
 Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along. More...
 
struct  MutexHolder
 
struct  PathTokenizer
 
class  Print
 
class  PurgePin
 Base class for reguesting directory space to obtain. More...
 
struct  ReadReqRH
 
struct  ReadRequest
 
class  ResourceMonitor
 
struct  SplitParser
 
class  Stats
 Statistics of cache utilisation by a File object. More...
 
struct  TmpConfiguration
 

Typedefs

using BlockList_i = std::list< Block * >::iterator
 
using BlockList_t = std::list< Block * >
 
using unlink_func = std::function< int(const std::string &)>
 
using vChunkRequest_i = std::vector< ChunkRequest >::iterator
 
using vChunkRequest_t = std::vector< ChunkRequest >
 
typedef std::vector< uint32_t > vCkSum_t
 

Enumerations

enum  CkSumCheck_e {
  CSChk_Unknown = -1 ,
  CSChk_None = 0 ,
  CSChk_Cache = 1 ,
  CSChk_Net = 2 ,
  CSChk_Both = 3 ,
  CSChk_TLS = 4
}
 

Functions

void OldStylePurgeDriver (DataFsPurgeshot &ps)
 
m_NDirectories PFC_DEFINE_TYPE_NON_INTRUSIVE (DirStateElement, m_dir_name, m_stats, m_usage, m_parent, m_daughters_begin, m_daughters_end) PFC_DEFINE_TYPE_NON_INTRUSIVE(DataFsSnapshot
 
 PFC_DEFINE_TYPE_NON_INTRUSIVE (DirStats, m_NumIos, m_Duration, m_BytesHit, m_BytesMissed, m_BytesBypassed, m_BytesWritten, m_StBlocksAdded, m_NCksumErrors, m_StBlocksRemoved, m_NFilesOpened, m_NFilesClosed, m_NFilesCreated, m_NFilesRemoved, m_NDirectoriesCreated, m_NDirectoriesRemoved) PFC_DEFINE_TYPE_NON_INTRUSIVE(DirUsage
 
long long UnlinkPurgeStateFilesInMap (FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)
 

Variables

m_NDirectories m_disk_total
 
m_NDirectories m_disk_used
 
m_NDirectories m_file_usage
 
 m_LastCloseTime
 
 m_LastOpenTime
 
m_NDirectories m_meta_total
 
m_NDirectories m_meta_used
 
 m_NFiles
 
 m_NFilesOpen
 
m_NDirectories m_sshot_stats_reset_time
 
 m_StBlocks
 
m_NDirectories m_usage_update_time
 
const char * trace_what_strings [] = {"","error ","warning ","info ","debug ","dump "}
 

Typedef Documentation

◆ BlockList_i

using XrdPfc::BlockList_i = typedef std::list<Block*>::iterator

Definition at line 167 of file XrdPfcFile.hh.

◆ BlockList_t

using XrdPfc::BlockList_t = typedef std::list<Block*>

Definition at line 166 of file XrdPfcFile.hh.

◆ unlink_func

using XrdPfc::unlink_func = typedef std::function<int(const std::string&)>

Definition at line 38 of file XrdPfcDirState.hh.

◆ vChunkRequest_i

using XrdPfc::vChunkRequest_i = typedef std::vector<ChunkRequest>::iterator

Definition at line 102 of file XrdPfcFile.hh.

◆ vChunkRequest_t

using XrdPfc::vChunkRequest_t = typedef std::vector<ChunkRequest>

Definition at line 101 of file XrdPfcFile.hh.

◆ vCkSum_t

typedef std::vector<uint32_t> XrdPfc::vCkSum_t

Definition at line 31 of file XrdPfcTypes.hh.

Enumeration Type Documentation

◆ CkSumCheck_e

Enumerator
CSChk_Unknown 
CSChk_None 
CSChk_Cache 
CSChk_Net 
CSChk_Both 
CSChk_TLS 

Definition at line 27 of file XrdPfcTypes.hh.

27 { CSChk_Unknown = -1, CSChk_None = 0, CSChk_Cache = 1, CSChk_Net = 2, CSChk_Both = 3,
28 CSChk_TLS = 4 // Only used during configuration parsing.
29};
@ CSChk_Unknown
@ CSChk_Cache

Function Documentation

◆ OldStylePurgeDriver()

void XrdPfc::OldStylePurgeDriver ( DataFsPurgeshot ps)

PurgePin

Default purge

Definition at line 99 of file XrdPfcPurge.cc.

100{
101 static const char *trc_pfx = "OldStylePurgeDriver ";
102 const auto &cache = Cache::TheOne();
103 const auto &conf = Cache::Conf();
104 auto &oss = *cache.GetOss();
105
106 time_t purge_start = time(0);
107
111 PurgePin *purge_pin = cache.GetPurgePin();
112 long long std_blocks_removed_by_pin = 0;
113 if (purge_pin)
114 {
115 // set dir stat for each path and calculate nBytes to recover for each path
116 // return total bytes to recover within the plugin
117 long long clearVal = purge_pin->GetBytesToRecover(ps);
118 if (clearVal)
119 {
120 TRACE(Debug, "PurgePin remove total " << clearVal << " bytes");
121 PurgePin::list_t &dpl = purge_pin->refDirInfos();
122 // iterate through the plugin paths
123 for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit)
124 {
125 TRACE(Debug, trc_pfx << "PurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes");
126
127 FPurgeState fps(ppit->nBytesToRecover, oss);
128 bool scan_ok = fps.TraverseNamespace(ppit->path.c_str());
129 if ( ! scan_ok) {
130 TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path);
131 continue;
132 }
133
134 fps.MoveListEntriesToMap();
135 std_blocks_removed_by_pin += UnlinkPurgeStateFilesInMap(fps, ppit->nBytesToRecover, ppit->path);
136 }
137 }
138 }
139
143
144 // check if the default pargue is still needed after purge pin
145 long long pin_removed_bytes = std_blocks_removed_by_pin * 512ll;
146 long long default_purge_blocks_removed = 0;
147 if (ps.m_bytes_to_remove > pin_removed_bytes)
148 {
149 // init default purge
150 long long bytes_to_remove = ps.m_bytes_to_remove - pin_removed_bytes;
151 FPurgeState purgeState(2 * bytes_to_remove, oss); // prepare twice more volume than required
152
153 if (ps.m_age_based_purge)
154 {
155 purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge);
156 }
157 if (conf.is_uvkeep_purge_in_effect())
158 {
159 purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep);
160 }
161
162 // Make a map of file paths, sorted by access time.
163 bool scan_ok = purgeState.TraverseNamespace("/");
164 if (!scan_ok)
165 {
166 TRACE(Error, trc_pfx << "default purge namespace traversal failed at top-directory, this should not happen.");
167 return;
168 }
169
170 TRACE(Debug, trc_pfx << "default purge usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes.");
171
172 purgeState.MoveListEntriesToMap();
173 default_purge_blocks_removed = UnlinkPurgeStateFilesInMap(purgeState, bytes_to_remove, "/");
174 }
175
176 // print the total summary
178 int purge_duration = time(0) - purge_start;
179 long long total_bytes_removed = (default_purge_blocks_removed + std_blocks_removed_by_pin) * 512ll;
180 TRACE(Info, trc_pfx << "Finished, removed total size " << total_bytes_removed << ", purge duration " << purge_duration);
181}
bool Debug
#define TRACE(act, x)
Definition XrdTrace.hh:63
Base class for reguesting directory space to obtain.
virtual long long GetBytesToRecover(const DataFsPurgeshot &)=0
std::vector< DirInfo > list_t
list_t & refDirInfos()
list_t::iterator list_i
long long UnlinkPurgeStateFilesInMap(FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)

References XrdPfc::Cache::Conf(), Debug, Error, XrdPfc::PurgePin::GetBytesToRecover(), XrdPfc::FPurgeState::getNBytesTotal(), XrdPfc::DataFsPurgeshot::m_age_based_purge, XrdPfc::DataFsPurgeshot::m_bytes_to_remove, XrdPfc::FPurgeState::MoveListEntriesToMap(), XrdPfc::PurgePin::refDirInfos(), XrdPfc::FPurgeState::setMinTime(), XrdPfc::FPurgeState::setUVKeepMinTime(), XrdPfc::Cache::TheOne(), TRACE, XrdPfc::FPurgeState::TraverseNamespace(), and UnlinkPurgeStateFilesInMap().

Referenced by XrdPfc::ResourceMonitor::perform_purge_task().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ PFC_DEFINE_TYPE_NON_INTRUSIVE() [1/2]

m_NDirectories XrdPfc::PFC_DEFINE_TYPE_NON_INTRUSIVE ( DirStateElement  ,
m_dir_name  ,
m_stats  ,
m_usage  ,
m_parent  ,
m_daughters_begin  ,
m_daughters_end   
)

◆ PFC_DEFINE_TYPE_NON_INTRUSIVE() [2/2]

XrdPfc::PFC_DEFINE_TYPE_NON_INTRUSIVE ( DirStats  ,
m_NumIos  ,
m_Duration  ,
m_BytesHit  ,
m_BytesMissed  ,
m_BytesBypassed  ,
m_BytesWritten  ,
m_StBlocksAdded  ,
m_NCksumErrors  ,
m_StBlocksRemoved  ,
m_NFilesOpened  ,
m_NFilesClosed  ,
m_NFilesCreated  ,
m_NFilesRemoved  ,
m_NDirectoriesCreated  ,
m_NDirectoriesRemoved   
)

◆ UnlinkPurgeStateFilesInMap()

long long XrdPfc::UnlinkPurgeStateFilesInMap ( FPurgeState purgeState,
long long  bytes_to_remove,
const std::string &  root_path 
)

Definition at line 24 of file XrdPfcPurge.cc.

25{
26 static const char *trc_pfx = "UnlinkPurgeStateFilesInMap ";
27
28 struct stat fstat;
29 int protected_cnt = 0;
30 int deleted_file_count = 0;
31 long long deleted_st_blocks = 0;
32 long long protected_st_blocks = 0;
33 long long st_blocks_to_remove = (bytes_to_remove >> 9) + 1ll;
34
35
36 const auto &cache = Cache::TheOne();
37 auto &resmon = Cache::ResMon();
38 auto &oss = *cache.GetOss();
39
40 TRACE(Info, trc_pfx << "Started, root_path = " << root_path << ", bytes_to_remove = " << bytes_to_remove);
41
42 // Loop over map and remove files with oldest values of access time.
43 for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it)
44 {
45 // Finish when enough space has been freed but not while age-based purging is in progress.
46 // Those files are marked with time-stamp = 0.
47 if (st_blocks_to_remove <= 0 && it->first != 0)
48 {
49 break;
50 }
51
52 std::string &infoPath = it->second.path;
53 std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen);
54
55 if (cache.IsFileActiveOrPurgeProtected(dataPath))
56 {
57 ++protected_cnt;
58 protected_st_blocks += it->second.nStBlocks;
59 TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks);
60 continue;
61 }
62
63 // remove info file
64 if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK)
65 {
66 oss.Unlink(infoPath.c_str());
67 TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size);
68 }
69 else
70 {
71 TRACE(Error, trc_pfx << "Can't locate file " << dataPath);
72 }
73
74 // remove data file
75 if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK)
76 {
77 st_blocks_to_remove -= it->second.nStBlocks;
78 deleted_st_blocks += it->second.nStBlocks;
79 ++deleted_file_count;
80
81 oss.Unlink(dataPath.c_str());
82 TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first);
83
84 resmon.register_file_purge(dataPath, it->second.nStBlocks);
85 }
86 }
87 if (protected_cnt > 0)
88 {
89 TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks);
90 }
91
92 TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks)
93
94 return deleted_st_blocks;
95}
#define XrdOssOK
Definition XrdOss.hh:50
#define fstat(a, b)
Definition XrdPosix.hh:62
#define stat(a, b)
Definition XrdPosix.hh:101
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:41

References Debug, Error, fstat, XrdPfc::FPurgeState::refMap(), XrdPfc::Cache::ResMon(), XrdPfc::Info::s_infoExtensionLen, stat, XrdPfc::Cache::TheOne(), TRACE, and XrdOssOK.

Referenced by OldStylePurgeDriver().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Variable Documentation

◆ m_disk_total

m_NDirectories XrdPfc::m_disk_total

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_disk_used

m_NDirectories XrdPfc::m_disk_used

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_file_usage

m_NDirectories XrdPfc::m_file_usage

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_LastCloseTime

XrdPfc::m_LastCloseTime

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_LastOpenTime

XrdPfc::m_LastOpenTime

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_meta_total

m_NDirectories XrdPfc::m_meta_total

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_meta_used

m_NDirectories XrdPfc::m_meta_used

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_NFiles

XrdPfc::m_NFiles

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_NFilesOpen

XrdPfc::m_NFilesOpen

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_sshot_stats_reset_time

m_NDirectories XrdPfc::m_sshot_stats_reset_time

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_StBlocks

XrdPfc::m_StBlocks

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_usage_update_time

m_NDirectories XrdPfc::m_usage_update_time

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ trace_what_strings

const char * XrdPfc::trace_what_strings = {"","error ","warning ","info ","debug ","dump "}

Definition at line 26 of file XrdPfcConfiguration.cc.

26{"","error ","warning ","info ","debug ","dump "};