For me in a similar situation works this way. First, create a new group (group_purchase_restrict) in Odoo or your module, and then declare the XML view like this:
<record id="purchase_order_form_mods" model="ir.ui.view">
<field name="name">purchase.order.form.mods</field>
<field name="model">purchase.order</field>
<field name="inherit_id" ref="purchase.purchase_order_form" />
<field name="arch" type="xml">
<xpath expr="//button[@name='purchase_confirm'][1]" position="attributes">
<attribute name="groups">purchase.group_purchase_user,purchase.group_purchase_manager</attribute>
</xpath>
</field>
</record>
<record id="purchase_order_form_mods_restrict" model="ir.ui.view">
<field name="name">purchase.order.form.mods.restrict</field>
<field name="model">purchase.order</field>
<field name="inherit_id" ref="purchase.purchase_order_form" />
<field name="groups_id" eval="[(6, 0, [ref('purchase_mods.group_purchase_restrict') ])]"/>
<field name="arch" type="xml">
<xpath expr="//button[@name='purchase_confirm'][1]" position="attributes">
<attribute name="invisible">1</attribute>
</xpath>
</field>
</record>
This way hide the button only for the users belonging to the new group (group_purchase_restrict).
I hope it helps