Step 1: Create Access right for Contact in security.xml file

<record model="ir.module.category" id="module_category_contact">
<field name="name">Contacts</field>
<field name="sequence">20</field>
<field name="parent_id" eval="False"/>
</record>

<record id="module_group_allow_contact_menu" model="ir.module.category">
<field name="name">Contacts</field>
<field name="parent_id" ref="module_category_contact"/>
</record>

<record id="group_allow_contact_menu_user" model="res.groups">
<field name="name">User: Own Documents Only</field>
<field name="category_id" ref="module_group_allow_contact_menu"/>
</record>

<record id="group_allow_contact_menu_admin" model="res.groups">
<field name="name">Administration</field>
<field name="category_id" ref="module_group_allow_contact_menu"/>
</record>

Step 2: Write below code into py file.
You may do this by using search_read method.
If you provide group_allow_contact_menu_user access rights to user then, that user can only see his partner.

from odoo import api, fields, models,

class ResPartner(models.Model):
_inherit = "res.partner"

@api.model
def search_read(self, domain=None, fields=None, offset=0, limit=None, order=None):
if self.env.user.has_group('Module Name.group_allow_contact_menu_user'):
domain += [('user_id', '=', self.env.user.id)]
return super(ResPartner, self).search_read(domain=domain, fields=fields, offset=offset, limit=limit, order=order)