'MFS - million files system, for huge file qty
'v.0.13
'(c) https://www.b4x.com/android/forum/members/peacemaker.1637/
Private Sub Class_Globals
Private SQL As SQL
Private OneHourFilesWatcher As FileWatcher
Private db_file_name As String = "mfs.db"
Private mRoot As String
Private mTimeOut As Int
Private const MAX_FILES_QTY As Long = 10000000 '10 million
Private FreePercentLimit As Int = 20
Public const FILE_NAME_PREFIX As String = "image_"
Public const FILE_NAME_EXTENSION As String = ".jpg"
Private LatestFolder As String
Private timOutdated As Timer
End Sub
'Initializes the object.
'RootFolder - folder where subfolders structure will be created
'StorageTimeOutTicks - milliseconds, after this time interval the oldest files will be deleted
Public Sub Initialize(RootFolder As String, dbfn As String, StorageTimeOutTicks As Long, StorageFreePercentLimit As Int) As Boolean
Log("RootFolder = " & RootFolder)
If dbfn = "" Then
db_file_name = "mfs.db"
Else
db_file_name = dbfn
End If
If StorageFreePercentLimit <= 0 Then
FreePercentLimit = 20
Else
FreePercentLimit = StorageFreePercentLimit
End If
timOutdated.Initialize("timOutdated", 2000)
timOutdated.Enabled = False
mTimeOut = StorageTimeOutTicks
If File.Exists(RootFolder, "") = False Then
Dim parent As String = File.GetFileParent(RootFolder)
Dim newfolder As String = File.GetName(RootFolder)
File.MakeDir(parent, newfolder)
End If
mRoot = RootFolder
Dim free_space As Long = Get_FreeSpace(RootFolder)
Dim file_size As Long = free_space / MAX_FILES_QTY
If file_size > 5 Then
SQL.InitializeSQLite(GetDBFolder("mfs"), db_file_name, True)
Prepare_DB
timOutdated.Enabled = True
OneHourFilesWatcher.Initialize("fw") 'Initialize with the event name
Return True
Else
Return False
End If
End Sub
Private Sub SleepMs(ms As Int)
Sleep(ms)
End Sub
Private Sub Prepare_DB
Dim ft As Map
ft.Initialize
ft.Put("id", DBUtils.DB_INTEGER)
ft.Put("name", DBUtils.DB_TEXT)
ft.Put("size", DBUtils.DB_INTEGER)
ft.Put("created", DBUtils.DB_INTEGER) 'milliseconds
ft.Put("lastmodified", DBUtils.DB_INTEGER) 'milliseconds
ft.Put("path", DBUtils.DB_TEXT) 'full path
DBUtils.CreateTable(SQL, "fs", ft, "id")
End Sub
'bytes
Private Sub Get_FreeSpace(Fold As String) As Long
'(ArrayList) [C:\, D:\, E:\, P:\, R:\]
'(MyMap) {C:\=126100873216, D:\=125829115904, E:\=104857595904, P:\=258932207616, R:\=104857595904}
'(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
'(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
'(MyMap) {C:\=96066768896, D:\=91721363456, E:\=83989377024, P:\=242732421120, R:\=83989377024}
If DetectOS = "windows" Then
Dim drive As String = Fold.SubString2(0, 3).ToUpperCase
Else If DetectOS = "linux" Then
Dim drive As String = "/"
End If
Private TotalDrives As List
TotalDrives.Initialize
TotalDrives=DiskUtils.GetDrives
'Log (TotalDrives)
Private TotalDriveCapacity As Map
TotalDriveCapacity.Initialize
TotalDriveCapacity=DiskUtils.GetDrivesCapacity
'Log (TotalDriveCapacity)
Private TotalUsableSpace As Map
TotalUsableSpace.Initialize
TotalUsableSpace=DiskUtils.GetUsableSpace
'Log (TotalUsableSpace)
Private TotalFreeSpace As Map
TotalFreeSpace.Initialize
TotalFreeSpace=DiskUtils.GetFreeSpace
'Log(TotalFreeSpace)
Private Writable As Map
Writable.Initialize
Writable=DiskUtils.Writable
Private UsedSpace As Map
UsedSpace.Initialize
UsedSpace=DiskUtils.GetUsedSpace
'Log(UsedSpace)
Dim u As Long = TotalUsableSpace.Get(drive)
Return u
End Sub
'%
Private Sub Get_FreeSpacePercent(Fold As String) As Long
'(ArrayList) [C:\, D:\, E:\, P:\, R:\]
'(MyMap) {C:\=126100873216, D:\=125829115904, E:\=104857595904, P:\=258932207616, R:\=104857595904}
'(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
'(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
'(MyMap) {C:\=96066768896, D:\=91721363456, E:\=83989377024, P:\=242732421120, R:\=83989377024}
If DetectOS = "windows" Then
Dim drive As String = Fold.SubString2(0, 3).ToUpperCase
Else If DetectOS = "linux" Then
Dim drive As String = "/"
End If
Private TotalDrives As List
TotalDrives.Initialize
TotalDrives=DiskUtils.GetDrives
'Log (TotalDrives)
Private TotalDriveCapacity As Map
TotalDriveCapacity.Initialize
TotalDriveCapacity=DiskUtils.GetDrivesCapacity
'Log (TotalDriveCapacity)
Private TotalUsableSpace As Map
TotalUsableSpace.Initialize
TotalUsableSpace=DiskUtils.GetUsableSpace
'Log (TotalUsableSpace)
Private TotalFreeSpace As Map
TotalFreeSpace.Initialize
TotalFreeSpace=DiskUtils.GetFreeSpace
'Log(TotalFreeSpace)
Private Writable As Map
Writable.Initialize
Writable=DiskUtils.Writable
Private UsedSpace As Map
UsedSpace.Initialize
UsedSpace=DiskUtils.GetUsedSpace
'Log(UsedSpace)
Dim u As Long = TotalUsableSpace.Get(drive)
Dim t As Long = TotalDriveCapacity.Get(drive)
Dim res As Int = (100-((t - u)/t) * 100)
'Log (res)
Return res
End Sub
Private Sub DetectOS As String
Dim os As String = GetSystemProperty("os.name", "").ToLowerCase
If os.Contains("win") Then
Return "windows"
Else If os.Contains("mac") Then
Return "mac"
Else
Return "linux"
End If
End Sub
Private Sub Get_Stamp(moment As Long) As String
Dim b As String, d, E As Long, f As Float
Dim prevDateF As String = DateTime.DateFormat
Dim prevTimeF As String = DateTime.TimeFormat
DateTime.DateFormat = "yyyyMMdd"
DateTime.TimeFormat = "HHmmss" ' "HH-mm-ss.SSS(Z)"
If moment = 0 Then
d = DateTime.Now
Else
d = moment
End If
E = d - DateTime.TimeParse(DateTime.Time(d))
f = E/DateTime.TicksPerSecond
b = NumberFormat2(f, 0, 3, 3, False)
Dim stamp As String = DateTime.DAte(d) & "_" & DateTime.Time(d) & b
DateTime.DateFormat = prevDateF
DateTime.TimeFormat = prevTimeF
Return stamp
End Sub
#if B4J
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Private Sub GetDBFolder (AppName As String) As String
Return File.DirData(AppName)
End Sub
#Else
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Private Sub GetDBFolder As String
#If B4A
Dim rp As RuntimePermissions
If File.ExternalWritable Then Return rp.GetSafeDirDefaultExternal("") Else Return File.DirInternal
#Else If B4i
Return File.DirDocuments
#End If
End Sub
#End If
Sub NewFile As String
Dim Now As Long = DateTime.Now
Dim fn As String = FILE_NAME_PREFIX & Get_Stamp(Now) & FILE_NAME_EXTENSION
Dim year As String = NumberFormat2(DateTime.GetYear(Now), 4, 0, 0, False)
Dim month As String = NumberFormat(DateTime.GetMonth(Now), 2, 0)
Dim day As String = NumberFormat(DateTime.GetDayOfMonth(Now), 2, 0)
Dim hour As String = NumberFormat(DateTime.GetHour(Now), 2, 0)
Dim delimiter As String = IIf(DetectOS = "windows", "\", "/")
Dim folder As String = mRoot & delimiter & year & delimiter & month & delimiter & day & delimiter & hour
folder = folder.Replace("//", "/").Replace("\\", "\")
If File.Exists(folder, "") = False Then
Dim parent As String = File.GetFileParent(folder)
Dim newfolder As String = File.GetName(folder)
File.MakeDir(parent, newfolder)
End If
Dim path As String = File.Combine(folder, fn)
If folder <> LatestFolder Then
' Log("folder = " & folder)
OneHourFilesWatcher.StopWatching
OneHourFilesWatcher.SetWatchList(Array As String(folder)) 'Set the current dir to be watched
OneHourFilesWatcher.StartWatching
SleepMs(0)
' Log("OneHourFilesWatcher started OK")
End If
LatestFolder = folder
'fw_CreationDetected(fn)
Return path
End Sub
Private Sub fw_CreationDetected(FileName As String)
Dim path As String = getPath(FileName)
'Log("path = " & path)
If path <> "" Then 'already exists
fw_ModificationDetected(FileName)
Return
End If
' Log("fw_CreationDetected = " & FileName)
Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
'Log("CreationDetected = " & DateTime.Now)
Dim Now As Long = File.LastModified(f, FileName) 'DateTime.Now
'Log("LastModified = " & Now)
Dim L As List:L.Initialize: Dim ft As Map:ft.Initialize
ft.Put("name", FileName)
ft.Put("size", File.Size(f, FileName))
ft.Put("created", Now) 'milliseconds
ft.Put("lastmodified", Now) 'milliseconds
ft.Put("path", File.Combine(f, FileName)) 'full path
L.Add(ft)
DBUtils.InsertMaps(SQL, "fs", L)
End Sub
Private Sub fw_DeletionDetected(FileName As String)
Log("fw_DeletionDetected = " & FileName)
Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
Dim where_map As Map:where_map.Initialize
where_map.Put("path", File.Combine(f, FileName))
DBUtils.DeleteRecord(SQL, "fs", where_map)
End Sub
Private Sub fw_ModificationDetected(FileName As String)
'Log("ModificationDetected = " & DateTime.Now)
Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
Dim beforemodified As Long = getLastmodified(FileName)
Dim LastModified As Long = File.LastModified(f, FileName)
Dim beforesize As Long = getSize(FileName)
Dim size As Long = File.Size(f, FileName)
If LastModified <> beforemodified Or beforesize <> size Then
Log("fw_ModificationDetected = " & FileName)
Dim where_map As Map:where_map.Initialize
where_map.Put("path", File.Combine(f, FileName))
Dim ft As Map:ft.Initialize
ft.Put("size", File.Size(f, FileName))
ft.Put("lastmodified", File.LastModified(f, FileName)) 'milliseconds
DBUtils.UpdateRecord2(SQL, "fs", ft, where_map)
End If
End Sub
Private Sub fw_Overflow
Log("fw_Overflow")
End Sub
Private Sub timOutdated_Tick
timOutdated.Enabled = False
Dim Now As Long = DateTime.Now
Dim limit As Long = Now - Abs(mTimeOut)
'Log("limit = " & limit)
Dim lim As Int = Get_FreeSpacePercent(mRoot)
If lim < FreePercentLimit Then
Dim L As List = getOldestFiles(1)
Dim t As Long = getLastmodified(L.Get(0))
limit = t + DateTime.TicksPerHour
End If
Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs WHERE created < '" & limit & "' ORDER by created")
Dim mapFolders As Map
mapFolders.Initialize
Do While r.NextRow
Dim p As String = r.GetString("path")
'Dim created As Long = r.GetLong("created")
'Log("created = " & created)
Dim fol As String = File.GetFileParent(p)
mapFolders.Put(fol, fol)
'Log(p)
If File.Exists(p, "") Then
Delete(p)
End If
Loop
'Log("all deleted !")
r.Close
For i = 0 To mapFolders.Size - 1
Dim fol As String = mapFolders.GetKeyAt(i)
Dim existing As Int = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs WHERE path LIKE '%" & fol & "%'")
If existing = 0 Then
File.Delete(fol, "")
End If
Next
timOutdated.Enabled = True
End Sub
Private Sub Delete_Img_Files_Only (folder1 As String, ext As String)
If folder1 = "" Then Return
Try
Dim L As List = File.ListFiles(folder1)
For i = 0 To L.Size - 1
Dim fn As String = L.Get(i)
If File.IsDirectory(folder1, fn) = False Then
If fn.ToLowerCase.EndsWith(ext.ToLowerCase) Then
File.Delete(folder1, fn)
End If
End If
Next
Log(folder1 & ": deleted all OK")
Catch
Log("Delete_All_Files_Only=" & fn & ": " & LastException)
End Try
End Sub
Private Sub DeleteFolder (folder As String, killself As Boolean)
For Each f As String In File.ListFiles(folder)
If File.IsDirectory(folder, f) Then
DeleteFolder(File.Combine(folder, f), True)
End If
File.Delete(folder, f)
Next
If killself Then
File.Delete(folder, "")
End If
End Sub
Sub Clear_All
OneHourFilesWatcher.StopWatching
Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs ORDER BY created DESC")
Do While r.NextRow
Dim p As String = r.GetString("path")
Dim fol As String = File.GetFileParent(p)
If File.Exists(fol, "") Then
DeleteFolder(fol, True)
End If
Loop
r.Close
DeleteFolder(mRoot, False)
DBUtils.ClearTable(SQL, "fs")
Log("All cleared !")
End Sub
Public Sub getPath(filename As String) As String
Try
Dim path As String = SQL.ExecQuerySingleResult("SELECT path FROM fs WHERE path LIKE '%" & filename & "%'")
Catch
Dim path As String = Null
End Try
If path = Null Then
path = ""
End If
Return path
End Sub
Public Sub getCreated(filename As String) As Long
Dim created As Long
Dim path As String = getPath(filename)
If path <> "" Then
created = SQL.ExecQuerySingleResult2("SELECT created FROM fs WHERE path = ?", Array As String(path))
End If
Return created
End Sub
Public Sub getLastmodified(filename As String) As Long
Dim Lastmodified As Long
Dim path As String = getPath(filename)
If path <> "" Then
Try
Lastmodified = SQL.ExecQuerySingleResult2("SELECT lastmodified FROM fs WHERE path = ?", Array As String(path))
Catch
Lastmodified = -1
End Try
End If
Return Lastmodified
End Sub
Public Sub getSize(filename As String) As Long
Dim Size As Long
Dim path As String = getPath(filename)
If path <> "" Then
Size = SQL.ExecQuerySingleResult2("SELECT size FROM fs WHERE path = ?", Array As String(path))
End If
Return Size
End Sub
Public Sub getLatestFiles(qty As Int) As List
Dim L As List
Try
L = DBUtils.ExecuteList(SQL, "SELECT path FROM fs ORDER BY lastmodified DESC", Null, qty)
Catch
Log("getLatestFiles.error = " & LastException.Message)
End Try
Return L
End Sub
Public Sub getOldestFiles(qty As Int) As List
Dim L As List
Try
L = DBUtils.ExecuteList(SQL, "SELECT path FROM fs ORDER BY lastmodified ASC", Null, qty)
Catch
Log("getOldestFiles.error = " & LastException.Message)
End Try
Return L
End Sub
Public Sub getTotalQty As Long
Dim a As Long = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs")
Return a
End Sub
Public Sub Delete(fullpath As String) As Boolean
Dim a As Boolean = File.Delete("", fullpath)
If a Then
SQL.ExecNonQuery("DELETE FROM fs WHERE path = '" & fullpath & "'")
End If
Return a
End Sub
Public Sub Delete2(filename As String) As Boolean
Dim path As String = getPath(filename)
Return Delete(path)
End Sub
. If I do the only solution is to open task manager and end the process. I once tried waiting over an hour but it never did finish loading.
