Assuming you are working in Odoo version 18.

Go to server_v18/addons/web/static/src/views/calendar/calendar_model.js

Find this block inside the loadFilterSection method:

const previousAllFilter = previousFilters.find((f) => f.type === "all");
filters.push(this.makeFilterAll(previousAllFilter, isUserOrPartner, filterInfo.label));

Replace it with:

const previousAllFilter = previousFilters.find((f) => f.type === "all");

// Check group before adding the "Everybody's calendars" filter
let shouldShowAllFilter = true;

if (isUserOrPartner) {
    shouldShowAllFilter = await user.hasGroup("your_module.group_everybody_calendar");
}

if (shouldShowAllFilter) {
    const allFilter = this.makeFilterAll(previousAllFilter, isUserOrPartner, filterInfo.label);
    if (allFilter) {
        filters.push(allFilter);
    }
}

Now only if the user is the memer of this group (e.g. "your_module_name.group_everybody_calendar"), then only this filter "Everybody's calendars" will be visible to them.