XRootD
Loading...
Searching...
No Matches
XrdPfcPurge.cc
Go to the documentation of this file.
1#include "XrdPfc.hh"
5#include "XrdPfcPurgePin.hh"
6#include "XrdPfcTrace.hh"
7
8#include "XrdOss/XrdOss.hh"
9
10#include <sys/time.h>
11
12namespace
13{
14 XrdSysTrace* GetTrace() { return XrdPfc::Cache::GetInstance().GetTrace(); }
15 const char *m_traceID = "ResourceMonitor";
16}
17
18//==============================================================================
19// OldStylePurgeDriver
20//==============================================================================
21namespace XrdPfc
22{
23
24long long UnlinkPurgeStateFilesInMap(FPurgeState& purgeState, long long bytes_to_remove, const std::string& root_path)
25{
26 static const char *trc_pfx = "UnlinkPurgeStateFilesInMap ";
27
28 struct stat fstat;
29 int protected_cnt = 0;
30 int deleted_file_count = 0;
31 long long deleted_st_blocks = 0;
32 long long protected_st_blocks = 0;
33 long long st_blocks_to_remove = (bytes_to_remove >> 9) + 1ll;
34
35
36 const auto &cache = Cache::TheOne();
37 auto &resmon = Cache::ResMon();
38 auto &oss = *cache.GetOss();
39
40 TRACE(Info, trc_pfx << "Started, root_path = " << root_path << ", bytes_to_remove = " << bytes_to_remove);
41
42 // Loop over map and remove files with oldest values of access time.
43 for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it)
44 {
45 // Finish when enough space has been freed but not while age-based purging is in progress.
46 // Those files are marked with time-stamp = 0.
47 if (st_blocks_to_remove <= 0 && it->first != 0)
48 {
49 break;
50 }
51
52 std::string &infoPath = it->second.path;
53 std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen);
54
55 if (cache.IsFileActiveOrPurgeProtected(dataPath))
56 {
57 ++protected_cnt;
58 protected_st_blocks += it->second.nStBlocks;
59 TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks);
60 continue;
61 }
62
63 // remove info file
64 if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK)
65 {
66 oss.Unlink(infoPath.c_str());
67 TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size);
68 }
69 else
70 {
71 TRACE(Error, trc_pfx << "Can't locate file " << dataPath);
72 }
73
74 // remove data file
75 if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK)
76 {
77 st_blocks_to_remove -= it->second.nStBlocks;
78 deleted_st_blocks += it->second.nStBlocks;
79 ++deleted_file_count;
80
81 oss.Unlink(dataPath.c_str());
82 TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first);
83
84 resmon.register_file_purge(dataPath, it->second.nStBlocks);
85 }
86 }
87 if (protected_cnt > 0)
88 {
89 TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks);
90 }
91
92 TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks)
93
94 return deleted_st_blocks;
95}
96
97// -------------------------------------------------------------------------------------
98
100{
101 static const char *trc_pfx = "OldStylePurgeDriver ";
102 const auto &cache = Cache::TheOne();
103 const auto &conf = Cache::Conf();
104 auto &oss = *cache.GetOss();
105
106 time_t purge_start = time(0);
107
111 PurgePin *purge_pin = cache.GetPurgePin();
112 long long std_blocks_removed_by_pin = 0;
113 if (purge_pin)
114 {
115 // set dir stat for each path and calculate nBytes to recover for each path
116 // return total bytes to recover within the plugin
117 long long clearVal = purge_pin->GetBytesToRecover(ps);
118 if (clearVal)
119 {
120 TRACE(Debug, "PurgePin remove total " << clearVal << " bytes");
121 PurgePin::list_t &dpl = purge_pin->refDirInfos();
122 // iterate through the plugin paths
123 for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit)
124 {
125 TRACE(Debug, trc_pfx << "PurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes");
126
127 FPurgeState fps(ppit->nBytesToRecover, oss);
128 bool scan_ok = fps.TraverseNamespace(ppit->path.c_str());
129 if ( ! scan_ok) {
130 TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path);
131 continue;
132 }
133
135 std_blocks_removed_by_pin += UnlinkPurgeStateFilesInMap(fps, ppit->nBytesToRecover, ppit->path);
136 }
137 }
138 }
139
143
144 // check if the default pargue is still needed after purge pin
145 long long pin_removed_bytes = std_blocks_removed_by_pin * 512ll;
146 long long default_purge_blocks_removed = 0;
147 if (ps.m_bytes_to_remove > pin_removed_bytes)
148 {
149 // init default purge
150 long long bytes_to_remove = ps.m_bytes_to_remove - pin_removed_bytes;
151 FPurgeState purgeState(2 * bytes_to_remove, oss); // prepare twice more volume than required
152
153 if (ps.m_age_based_purge)
154 {
155 purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge);
156 }
157 if (conf.is_uvkeep_purge_in_effect())
158 {
159 purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep);
160 }
161
162 // Make a map of file paths, sorted by access time.
163 bool scan_ok = purgeState.TraverseNamespace("/");
164 if (!scan_ok)
165 {
166 TRACE(Error, trc_pfx << "default purge namespace traversal failed at top-directory, this should not happen.");
167 return;
168 }
169
170 TRACE(Debug, trc_pfx << "default purge usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes.");
171
172 purgeState.MoveListEntriesToMap();
173 default_purge_blocks_removed = UnlinkPurgeStateFilesInMap(purgeState, bytes_to_remove, "/");
174 }
175
176 // print the total summary
178 int purge_duration = time(0) - purge_start;
179 long long total_bytes_removed = (default_purge_blocks_removed + std_blocks_removed_by_pin) * 512ll;
180 TRACE(Info, trc_pfx << "Finished, removed total size " << total_bytes_removed << ", purge duration " << purge_duration);
181}
182
183} // end namespace XrdPfc
#define XrdOssOK
Definition XrdOss.hh:50
#define fstat(a, b)
Definition XrdPosix.hh:62
#define stat(a, b)
Definition XrdPosix.hh:101
bool Debug
#define TRACE(act, x)
Definition XrdTrace.hh:63
static const Configuration & Conf()
Definition XrdPfc.cc:134
XrdSysTrace * GetTrace()
Definition XrdPfc.hh:283
static ResourceMonitor & ResMon()
Definition XrdPfc.cc:135
static Cache & GetInstance()
Singleton access.
Definition XrdPfc.cc:132
static const Cache & TheOne()
Definition XrdPfc.cc:133
void setUVKeepMinTime(time_t min_time)
void setMinTime(time_t min_time)
bool TraverseNamespace(const char *root_path)
long long getNBytesTotal() const
Status of cached file. Can be read from and written into a binary file.
Definition XrdPfcInfo.hh:41
static const size_t s_infoExtensionLen
Base class for reguesting directory space to obtain.
virtual long long GetBytesToRecover(const DataFsPurgeshot &)=0
std::vector< DirInfo > list_t
list_t & refDirInfos()
list_t::iterator list_i
void OldStylePurgeDriver(DataFsPurgeshot &ps)
long long UnlinkPurgeStateFilesInMap(FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)