Worked examples Preview

Each example states the handler settings it needs, because a script is only half of the answer. All of them have been written against the behavior documented in this book, including the traps.

Refuse uploads by extension

A policy gate. Cheap, synchronous, safe in front of a transfer.

Setting	Value
Event	Before a file is uploaded
Applies to	One virtual file system, while testing
If the script fails	Block the operation
Timeout	5000

var BLOCKED = [".exe", ".dll", ".scr", ".bat", ".cmd", ".ps1", ".js", ".vbs"];

var ext = ExtractExt(CtxRelPath()).toLowerCase();
if (BLOCKED.indexOf(ext) >= 0) {
    Log.Warn("refused " + ext + " upload by " + CtxUsername() + ": " + CtxRelPath());
    Exit(1);
}

Refuse content that lies about its extension

Same idea, but checks the bytes. Needs an export to local disk, so it is heavier; keep the timeout generous and consider running it after upload instead of before.

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	30000

var vfs = GetCurrentVFS();
if (vfs === null) { Exit(0); }

// A name we invented: nothing the user chose reaches the file system or a command line.
var local = "/var/scan/" + ShortUID() + ExtractExt(CtxRelPath()).toLowerCase();

if (!vfs.ExportFile(CtxRelPath(), local).Ok()) {
    Log.Error("could not export " + CtxRelPath() + " for inspection");
    Exit(0);
}

var mime = FileType(local);
SecureErase(local);

var DANGEROUS = ["application/x-msdownload", "application/x-executable", "application/x-sharedlib"];
if (DANGEROUS.indexOf(mime) >= 0) {
    Log.Error("executable content uploaded as " + CtxRelPath() + " by " + CtxUsername() +
              " (detected " + mime + "); removing");
    var rm = vfs.Remove(CtxRelPath());
    if (!rm.Ok()) {
        Log.Error("could not remove it: " + rm.ErrorMsg());
    }
}

Why this is an after handler

A before handler runs before the file exists, so there is nothing to inspect. Inspecting content means letting the upload finish and then dealing with what arrived.

Hide working files from listings

Setting	Value
Event	Filter a folder listing
Applies to	The virtual file system you want it on

Session.RemoveFromDirList("*.tmp");
Session.RemoveFromDirList("*.part");
Session.RemoveFromDirList("*.filepart");
Session.RemoveFromDirList(".DS_Store");
Session.RemoveFromDirList("[Tt]humbs.db");
Session.RemoveFromDirList(".*");           // dotfiles

This hides, it does not protect

A user who knows the name can still open the file. See Permissions.

File an audit line into a database

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	15000

var cli = new SqlCli().Driver("pg").ConnString(
    "host=db.internal port=5432 user=sc_conn password=" + dbPassword +
    " dbname=transfers sslmode=require"
);

if (!cli.Connect()) {
    Log.Error("audit database unreachable; the upload stands regardless");
    Exit(0);
}

cli.Exec(
    "INSERT INTO uploads (username, vfs, path, at) VALUES (?, ?, ?, ?)",
    [CtxUsername(), CtxVFSName(), CtxRelPath(), FormatDateTime("YYYY-MM-DD HH:mm:ss")]
);
cli.Close();

The Connector already keeps an audit trail

This is for feeding your systems. The Connector's own signed log records every operation whether you script anything or not, and unlike a script it cannot be turned off or fall behind.

Notify a chat channel

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	15000

// Only announce arrivals in the folder people are watching.
if (CtxRelPath().indexOf("/dropbox/") !== 0) { Exit(0); }

var msg = CtxUsername() + " uploaded " + ExtractName(CtxRelPath()) +
          " to " + CtxVFSName() + CtxRelPath();

if (!SendToSlackWebHook(slackWebhookUrl, msg, "SFTP.cloud", ":inbox_tray:")) {
    Log.Warn("Slack did not accept the notification");
}

Call your own webhook

The body is sent exactly as JSON.stringify produced it, with an application/json content type.

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	20000

var hc = new HttpCli();
var res = hc.Url("https://intake.internal.example.com/sftp-events")
            .Header("X-Source", "sftp-cloud-connector")
            .Bearer(intakeToken)
            .Timeout(10)
            .ReqBody(JSON.stringify({
                event: EventHandler(),
                user: CtxUsername(),
                vfs: CtxVFSName(),
                path: CtxRelPath(),
                at: FormatDateTime("YYYY-MM-DDTHH:mm:ss"),
                priority: HandlerPriority()
            }))
            .Post();

if (!res.Ok()) {
    Log.Error("intake unreachable: " + res.ErrorMsg());
} else if (res.StatusCode() < 200 || res.StatusCode() >= 300) {
    Log.Error("intake answered " + res.StatusCode() + ": " + res.BodyAsString());
}

Push a finished upload to a partner over SFTP

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	120000

if (CtxRelPath().indexOf("/outbound/") !== 0) { Exit(0); }

var vfs = GetCurrentVFS();
if (vfs === null) { Exit(0); }

var name  = ExtractName(CtxRelPath());
var local = "/var/staging/" + ShortUID() + "-" + name;

if (!vfs.ExportFile(CtxRelPath(), local).Ok()) {
    Log.Error("could not stage " + CtxRelPath());
    Exit(0);
}

var cli = new SftpClient();
cli.Host          = "sftp.partner.com:22";
cli.User          = "acme";
cli.KeyFile       = "/opt/keys/partner_ed25519";
cli.HostKeySHA256 = "SHA256:8ZQx1n2Rk8yWv4hQ7LdT3mFbC5aJ9pXsYuOeR6iKgNc";

cli.Options.UploadPolicy       = AlwaysOverwrite;
cli.Options.UploadWithTempName = true;      // the partner never sees a half written file

if (cli.Connect()) {
    if (cli.Upload(local, "/incoming")) {
        Log.Info("delivered " + name + " to the partner");
    } else {
        Log.Error("delivery of " + name + " failed");
    }
    cli.Close();
} else {
    Log.Error("could not connect to the partner SFTP server");
}

SecureErase(local);

Consider MoveToVFS instead

If the destination can be expressed as a virtual file system, which covers S3, Azure, Google Cloud Storage and SFTP, MoveToVFS does the same job with no local staging, no plaintext on disk and no cleanup to forget.

Archive to a bucket and remove the original

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	120000

if (CtxRelPath().indexOf("/archive-me/") !== 0) { Exit(0); }

var origin = GetCurrentVFS();
if (origin === null) { Exit(0); }

var bucket = new VirtualFS(
    VfsTypeS3,
    "s3://acme-archive?region=eu-west-1",
    JSON.stringify({
        access_key: s3AccessKey,
        access_secret: s3SecretKey,
        region: "eu-west-1"
    })
);

var target = "/" + FormatDateTime("YYYY/MM/DD") + "/" + ExtractName(CtxRelPath());
var resp = origin.MoveToVFS(CtxRelPath(), bucket, target);

if (resp.Ok()) {
    Log.Info("archived " + CtxRelPath() + " to " + target);
} else {
    Log.Error("archive failed, the file is still in place: " + resp.ErrorMsg());
}

Keep a folder under a size cap

Setting	Value
Event	After a file is uploaded
Run in the background	On
Timeout	60000

var CAP = 50 * 1024 * 1024 * 1024;   // 50 GiB
var FOLDER = "/scratch";

if (CtxRelPath().indexOf(FOLDER + "/") !== 0) { Exit(0); }

var vfs = GetCurrentVFS();
if (vfs === null) { Exit(0); }

var size = vfs.TreeSize(FOLDER);
if (!size.Ok()) {
    Log.Error("could not measure " + FOLDER + ": " + size.ErrorMsg());
    Exit(0);
}
if (size.Size() <= CAP) { Exit(0); }

Log.Warn(FOLDER + " is " + size.Size() + " bytes, over the cap; pruning oldest first");

var listing = vfs.ReadDir(FOLDER, SortByTime, SortAsc);
if (!listing.Ok()) { Exit(0); }

var total = size.Size();
var items = listing.Infos();
for (var i = 0; i < items.length && total > CAP; i++) {
    if (items[i].Type !== "FILE") { continue; }
    var victim = FOLDER + "/" + ExtractName(items[i].Name);
    var rm = vfs.Remove(victim);
    if (rm.Ok()) {
        total -= items[i].Size;
        Log.Info("pruned " + victim);
    } else {
        Log.Error("could not prune " + victim + ": " + rm.ErrorMsg());
    }
}

Test a pruning script somewhere that does not matter

This one deletes files. Run it against a scratch virtual file system, with the cap set low enough to trigger, and read the log before you point it anywhere real. Remember that GetCurrentVFS() ignores the user's permissions: it will happily delete something the person who triggered it could never have touched.

Guard a rename

before.rename is the only event with two paths.

Setting	Value
Event	Before a rename or move
If the script fails	Block the operation
Timeout	5000

var from = CtxRelPath();
var to   = CtxRelTargetPath();

// Do not let anything be renamed out of the archive folder.
if (from.indexOf("/archive/") === 0 && to.indexOf("/archive/") !== 0) {
    Log.Warn(CtxUsername() + " tried to move " + from + " out of the archive to " + to);
    Exit(1);
}

// Do not let an extension be changed to something we refuse on upload.
var newExt = ExtractExt(to).toLowerCase();
if (newExt === ".exe" || newExt === ".dll") {
    Log.Warn(CtxUsername() + " tried to rename " + from + " to " + to);
    Exit(1);
}

A time window

Setting	Value
Event	Before a file is uploaded
If the script fails	Block the operation

var hour = parseInt(FormatDateTime("HH"), 10);

// The bulk load window is 22:00 to 06:00, Connector local time.
if (CtxVFSName() === "BulkLoad" && hour >= 6 && hour < 22) {
    Log.Warn("out of window upload refused for " + CtxUsername() + ": " + CtxRelPath());
    Exit(1);
}

Which clock

FormatDateTime() with no date argument uses the Connector host's local time, not the user's and not the Portal's. If your users are in another timezone, say so in the message you log, and consider comparing in UTC by setting the host's timezone deliberately.

A template to start from

// What this does:
// Bound to:      <event>
// Scope:         <virtual file system>
// Fail mode:     <block or proceed>
// Background:    <yes or no>

Log.Debug("[" + ScriptName() + "] " + JSON.stringify(EventCtx()));

var vfs = GetCurrentVFS();
if (vfs === null) {
    Log.Warn("no virtual file system on this event, nothing to do");
    Exit(0);
}

// Narrow early: do nothing on the operations this script is not about.
if (CtxRelPath().indexOf("/the-folder-i-care-about/") !== 0) {
    Exit(0);
}

try {
    // the actual work
} catch (e) {
    Log.Error("[" + ScriptName() + "] failed: " + e);
    Exit(1);        // on a before handler this refuses; on an after handler it only logs
}